Build Hum recording and generation Space
Browse files- .dockerignore +7 -0
- .gitignore +7 -0
- Dockerfile +31 -0
- README.md +35 -5
- app.py +82 -0
- logic.py +71 -0
- pipeline.py +244 -0
- sheet_worker.py +56 -0
- test_logic.py +56 -0
- yue_worker.py +51 -0
.dockerignore
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
__pycache__/
|
| 2 |
+
*.pyc
|
| 3 |
+
.pytest_cache/
|
| 4 |
+
.venv/
|
| 5 |
+
.env
|
| 6 |
+
*.wav
|
| 7 |
+
*.mp3
|
.gitignore
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
__pycache__/
|
| 2 |
+
*.pyc
|
| 3 |
+
.pytest_cache/
|
| 4 |
+
.venv/
|
| 5 |
+
.env
|
| 6 |
+
*.wav
|
| 7 |
+
*.mp3
|
Dockerfile
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
FROM python:3.12-slim
|
| 2 |
+
|
| 3 |
+
ENV DEBIAN_FRONTEND=noninteractive \
|
| 4 |
+
PYTHONUNBUFFERED=1 \
|
| 5 |
+
HF_HOME=/home/user/.cache/huggingface \
|
| 6 |
+
GRADIO_ANALYTICS_ENABLED=False
|
| 7 |
+
|
| 8 |
+
RUN apt-get update && apt-get install -y --no-install-recommends \
|
| 9 |
+
ffmpeg fluidsynth libsndfile1 libgomp1 && \
|
| 10 |
+
rm -rf /var/lib/apt/lists/*
|
| 11 |
+
|
| 12 |
+
# SheetSage2's published code uses the older Transformers API. Separate
|
| 13 |
+
# processes keep that runtime independent of YuE2's pinned dependencies.
|
| 14 |
+
RUN python -m venv /opt/sheet && /opt/sheet/bin/pip install --no-cache-dir \
|
| 15 |
+
torch==2.8.0 torchaudio==2.8.0 transformers==4.45.2 \
|
| 16 |
+
numpy==1.26.4 scipy==1.15.3 huggingface-hub==0.36.0 \
|
| 17 |
+
safetensors==0.7.0 soundfile==0.13.1 pretty_midi==0.2.11 mido==1.3.3
|
| 18 |
+
|
| 19 |
+
RUN python -m venv /opt/yue && /opt/yue/bin/pip install --no-cache-dir \
|
| 20 |
+
'https://huggingface.co/m-a-p/YuE2-3B/resolve/29b3558dd46954a0cd9021dc76d5c91864a0f1c7/yue2_infer-0.1.5-py3-none-any.whl'
|
| 21 |
+
|
| 22 |
+
RUN python -m venv /opt/ui && /opt/ui/bin/pip install --no-cache-dir \
|
| 23 |
+
gradio==5.49.1 numpy==1.26.4 scipy==1.15.3 soundfile==0.13.1 \
|
| 24 |
+
librosa==0.11.0 pretty_midi==0.2.11 mido==1.3.3
|
| 25 |
+
|
| 26 |
+
RUN useradd -m -u 1000 user
|
| 27 |
+
WORKDIR /home/user/app
|
| 28 |
+
COPY --chown=user:user . /home/user/app
|
| 29 |
+
USER user
|
| 30 |
+
EXPOSE 7860
|
| 31 |
+
CMD ["/opt/ui/bin/python", "app.py"]
|
README.md
CHANGED
|
@@ -1,10 +1,40 @@
|
|
| 1 |
---
|
| 2 |
title: Hum
|
| 3 |
-
emoji:
|
| 4 |
-
colorFrom:
|
| 5 |
-
colorTo:
|
| 6 |
sdk: docker
|
| 7 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8 |
---
|
| 9 |
|
| 10 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
title: Hum
|
| 3 |
+
emoji: 🎵
|
| 4 |
+
colorFrom: indigo
|
| 5 |
+
colorTo: blue
|
| 6 |
sdk: docker
|
| 7 |
+
app_port: 7860
|
| 8 |
+
fullWidth: true
|
| 9 |
+
short_description: Turn a hummed melody into an instrument, or explore YuE2.
|
| 10 |
+
models:
|
| 11 |
+
- m-a-p/SheetSage2
|
| 12 |
+
- m-a-p/MERT-v2-FullSong
|
| 13 |
+
- m-a-p/YuE2-3B
|
| 14 |
+
- m-a-p/YuE2-Vae
|
| 15 |
---
|
| 16 |
|
| 17 |
+
# Hum
|
| 18 |
+
|
| 19 |
+
Upload a short hum or record one in the browser. The default mode extracts a
|
| 20 |
+
source-timed melody with SheetSage2 and renders **one** SoundFont instrument.
|
| 21 |
+
The continuous-F0 mode is a diagnostic synth following the voice pitch and
|
| 22 |
+
amplitude. YuE2 solo and arrangement modes use the SheetSage2 ABC score, but
|
| 23 |
+
are experimental: YuE2 can introduce voices or other sounds and does not
|
| 24 |
+
preserve source timing exactly.
|
| 25 |
+
|
| 26 |
+
Output audio is downloadable 320 kbit/s CBR MP3. Source recordings remain in
|
| 27 |
+
the Space's temporary job directory; they are not committed or used to train
|
| 28 |
+
models. The Space should remain private while testing personal recordings.
|
| 29 |
+
|
| 30 |
+
GPU recommendation: L40S 48 GB, one queued inference at a time, with idle
|
| 31 |
+
sleep enabled. Models are downloaded at first use and cached on the Space's
|
| 32 |
+
ephemeral disk. A cold restart can require another download.
|
| 33 |
+
|
| 34 |
+
Model licenses and source: [SheetSage2](https://huggingface.co/m-a-p/SheetSage2),
|
| 35 |
+
[MERT-v2-FullSong](https://huggingface.co/m-a-p/MERT-v2-FullSong),
|
| 36 |
+
[YuE2-3B](https://huggingface.co/m-a-p/YuE2-3B),
|
| 37 |
+
[YuE2-Vae](https://huggingface.co/m-a-p/YuE2-Vae).
|
| 38 |
+
|
| 39 |
+
This is a research prototype; a single owner-reviewed recording is development
|
| 40 |
+
evidence, not a general accuracy guarantee.
|
app.py
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Hum research Space: record, choose a rendering path, hear the result."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import gradio as gr
|
| 6 |
+
|
| 7 |
+
from logic import INSTRUMENTS
|
| 8 |
+
from pipeline import MODES, generate
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
SOLO_STYLE = (
|
| 12 |
+
"instrumental solo {instrument}, clear single melody, intimate dry recording, "
|
| 13 |
+
"natural instrumental tone"
|
| 14 |
+
)
|
| 15 |
+
ARRANGEMENT_STYLE = (
|
| 16 |
+
"instrumental chamber pop, {instrument} melody, acoustic bass, "
|
| 17 |
+
"brushed drums, soft strings, warm intimate mood"
|
| 18 |
+
)
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def suggest_prompt(mode: str, instrument: str) -> str:
|
| 22 |
+
name = INSTRUMENTS[instrument][1]
|
| 23 |
+
template = ARRANGEMENT_STYLE if mode == "YuE2 · композиция (эксперимент)" else SOLO_STYLE
|
| 24 |
+
return template.format(instrument=name)
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def run(audio: str | None, mode: str, instrument: str, style: str, seed: int,
|
| 28 |
+
progress: gr.Progress = gr.Progress(track_tqdm=False)):
|
| 29 |
+
try:
|
| 30 |
+
return generate(audio, mode, instrument, style, seed, progress)
|
| 31 |
+
except Exception as error:
|
| 32 |
+
raise gr.Error(str(error)) from error
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
CSS = """
|
| 36 |
+
.gradio-container {max-width: 1000px !important; margin: auto !important;}
|
| 37 |
+
#hero {padding: 20px 4px 10px 4px;}
|
| 38 |
+
#hero h1 {font-size: clamp(2.5rem, 6vw, 4.5rem); letter-spacing: -.045em; margin: 0;}
|
| 39 |
+
#hero p {font-size: 1.08rem; max-width: 720px; line-height: 1.55; color: #566179;}
|
| 40 |
+
.note {border: 1px solid #d9e0ed; border-radius: 16px; padding: 14px 18px;
|
| 41 |
+
background: linear-gradient(120deg,#f5f7ff,#f1fbfc); color: #3e4b64;}
|
| 42 |
+
"""
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
with gr.Blocks(title="Hum · напев в инструмент", css=CSS, theme=gr.themes.Soft()) as demo:
|
| 46 |
+
gr.HTML("""<div id='hero'><h1>Hum <span style='color:#6474db'>♪</span></h1>
|
| 47 |
+
<p>Напойте фразу в микрофон или загрузите короткую запись. Hum извлечёт
|
| 48 |
+
мелодию и сыграет её инструментом. Можно отдельно сравнить непрерывный
|
| 49 |
+
контур и экспериментальную генерацию YuE2.</p></div>""")
|
| 50 |
+
with gr.Row():
|
| 51 |
+
with gr.Column(scale=5):
|
| 52 |
+
audio = gr.Audio(label="Ваш напев", sources=["microphone", "upload"],
|
| 53 |
+
type="filepath", format="wav")
|
| 54 |
+
mode = gr.Radio(list(MODES), value="SheetSage2 · один инструмент",
|
| 55 |
+
label="Режим")
|
| 56 |
+
instrument = gr.Dropdown(list(INSTRUMENTS), value="Флейта", label="Инструмент")
|
| 57 |
+
with gr.Accordion("Настройки YuE2", open=False):
|
| 58 |
+
style = gr.Textbox(label="Style prompt", lines=2,
|
| 59 |
+
value=suggest_prompt("YuE2 · соло (эксперимент)", "Флейта"),
|
| 60 |
+
info="Используется только в режимах YuE2; поле lyrics пустое.")
|
| 61 |
+
seed = gr.Number(label="Seed", value=4096, precision=0)
|
| 62 |
+
go = gr.Button("Превратить в музыку", variant="primary", size="lg")
|
| 63 |
+
with gr.Column(scale=5):
|
| 64 |
+
result = gr.Audio(label="Результат · MP3 320 кбит/с", interactive=False)
|
| 65 |
+
with gr.Row():
|
| 66 |
+
midi = gr.File(label="Ноты · MIDI")
|
| 67 |
+
abc = gr.File(label="Ноты · ABC")
|
| 68 |
+
details = gr.JSON(label="Детали запуска")
|
| 69 |
+
gr.HTML("""<div class='note'><b>Текущая цель:</b> один инструмент и
|
| 70 |
+
приблизительно исходное время — режим SheetSage2. Непрерывный
|
| 71 |
+
контур — простой диагностический синтезатор. YuE2 может изменить
|
| 72 |
+
длительность, добавить инструменты или человеческие звуки;
|
| 73 |
+
отсутствие вокала в нём не гарантируется.</div>""")
|
| 74 |
+
mode.change(suggest_prompt, inputs=[mode, instrument], outputs=style)
|
| 75 |
+
instrument.change(suggest_prompt, inputs=[mode, instrument], outputs=style)
|
| 76 |
+
go.click(run, inputs=[audio, mode, instrument, style, seed],
|
| 77 |
+
outputs=[result, midi, abc, details], show_progress="full")
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
if __name__ == "__main__":
|
| 81 |
+
demo.queue(max_size=8, default_concurrency_limit=1)
|
| 82 |
+
demo.launch(server_name="0.0.0.0", server_port=7860, show_error=True)
|
logic.py
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Small deterministic contracts shared by the Space workers."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
from pathlib import Path
|
| 6 |
+
import re
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
MAX_SECONDS = 35.0
|
| 10 |
+
INSTRUMENTS = {
|
| 11 |
+
"Флейта": (73, "flute"),
|
| 12 |
+
"Кларнет": (71, "clarinet"),
|
| 13 |
+
"Фортепиано": (0, "acoustic piano"),
|
| 14 |
+
"Виолончель": (42, "cello"),
|
| 15 |
+
}
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def instrumental_abc(native: str, keep_chords: bool = False) -> str:
|
| 19 |
+
"""Move the native Vocal melody into Ins, failing on unknown ABC layout."""
|
| 20 |
+
lines = native.splitlines()
|
| 21 |
+
header = []
|
| 22 |
+
body = []
|
| 23 |
+
in_body = False
|
| 24 |
+
for line in lines:
|
| 25 |
+
if line == "V: Vocal":
|
| 26 |
+
in_body = True
|
| 27 |
+
(body if in_body else header).append(line)
|
| 28 |
+
if not body or not any(line.startswith("V: Vocal ") for line in header):
|
| 29 |
+
raise ValueError("SheetSage2 ABC has no expected Vocal melody")
|
| 30 |
+
header = [line for line in header if not line.startswith(("V: Vocal ", "V: Ins "))]
|
| 31 |
+
key_index = next((i for i, line in enumerate(header) if line.startswith("K:")), None)
|
| 32 |
+
if key_index is None:
|
| 33 |
+
raise ValueError("ABC has no key signature")
|
| 34 |
+
header.insert(key_index, 'V: Ins clef=treble name="Instrumental Melody" snm="Ins"')
|
| 35 |
+
out = []
|
| 36 |
+
index = 0
|
| 37 |
+
moved = 0
|
| 38 |
+
while index < len(body):
|
| 39 |
+
line = body[index]
|
| 40 |
+
if line.startswith("%") or not line.strip():
|
| 41 |
+
out.append(line)
|
| 42 |
+
index += 1
|
| 43 |
+
continue
|
| 44 |
+
if (index + 3 >= len(body) or body[index] != "V: Vocal"
|
| 45 |
+
or body[index + 2] != "V: Ins"
|
| 46 |
+
or not re.fullmatch(r"Z\d*\|", body[index + 3])):
|
| 47 |
+
raise ValueError("Unexpected SheetSage2 ABC voice layout")
|
| 48 |
+
melody = body[index + 1]
|
| 49 |
+
if not keep_chords:
|
| 50 |
+
melody = re.sub(r'"[^"\n]*"', "", melody)
|
| 51 |
+
out.extend(("V: Ins", melody))
|
| 52 |
+
moved += 1
|
| 53 |
+
index += 4
|
| 54 |
+
if not moved:
|
| 55 |
+
raise ValueError("No melody bars in SheetSage2 ABC")
|
| 56 |
+
result = "\n".join(header + out) + "\n"
|
| 57 |
+
if "Vocal" in result or (not keep_chords and any('"' in line for line in out)):
|
| 58 |
+
raise ValueError("Vocal label or unexpected chord survived conversion")
|
| 59 |
+
return result
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def validate_duration(seconds: float) -> None:
|
| 63 |
+
if not 0.4 <= seconds <= MAX_SECONDS:
|
| 64 |
+
raise ValueError(f"Нужен напев от 0,4 до {MAX_SECONDS:g} секунд; сейчас {seconds:.1f} с.")
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def work_file(root: Path, name: str) -> Path:
|
| 68 |
+
path = root / name
|
| 69 |
+
if path.parent != root or path.name != name:
|
| 70 |
+
raise ValueError("Invalid work file name")
|
| 71 |
+
return path
|
pipeline.py
ADDED
|
@@ -0,0 +1,244 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Audio ingest, source-timed renderers, and isolated GPU workers."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import hashlib
|
| 6 |
+
import json
|
| 7 |
+
import os
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
import subprocess
|
| 10 |
+
import sys
|
| 11 |
+
import tempfile
|
| 12 |
+
import time
|
| 13 |
+
|
| 14 |
+
from logic import INSTRUMENTS, instrumental_abc, validate_duration
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
ROOT = Path(__file__).resolve().parent
|
| 18 |
+
SHEET_PYTHON = Path(os.environ.get("HUM_SHEET_PYTHON", "/opt/sheet/bin/python"))
|
| 19 |
+
YUE_PYTHON = Path(os.environ.get("HUM_YUE_PYTHON", "/opt/yue/bin/python"))
|
| 20 |
+
MODES = {
|
| 21 |
+
"SheetSage2 · один инструмент": "sheet",
|
| 22 |
+
"Непрерывная высота · synth": "contour",
|
| 23 |
+
"YuE2 · соло (эксперимент)": "yue_solo",
|
| 24 |
+
"YuE2 · композиция (эксперимент)": "yue_arrangement",
|
| 25 |
+
}
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def command(args: list[str], log: Path, timeout: int = 900) -> None:
|
| 29 |
+
with log.open("a") as stream:
|
| 30 |
+
process = subprocess.run(args, stdout=stream, stderr=subprocess.STDOUT,
|
| 31 |
+
text=True, timeout=timeout, check=False)
|
| 32 |
+
if process.returncode:
|
| 33 |
+
tail = "\n".join(log.read_text(errors="replace").splitlines()[-18:])
|
| 34 |
+
raise RuntimeError(f"Шаг завершился с кодом {process.returncode}:\n{tail}")
|
| 35 |
+
|
| 36 |
+
|
| 37 |
+
def probe(source: Path) -> float:
|
| 38 |
+
if not source.is_file() or source.stat().st_size > 30 * 1024 * 1024:
|
| 39 |
+
raise ValueError("Файл отсутствует или больше 30 МБ.")
|
| 40 |
+
result = subprocess.run([
|
| 41 |
+
"ffprobe", "-v", "error", "-select_streams", "a:0",
|
| 42 |
+
"-show_entries", "stream=duration", "-of", "json", str(source),
|
| 43 |
+
], capture_output=True, text=True, check=True)
|
| 44 |
+
streams = json.loads(result.stdout).get("streams", [])
|
| 45 |
+
if not streams:
|
| 46 |
+
raise ValueError("В файле нет звуковой дорожки.")
|
| 47 |
+
duration = float(streams[0]["duration"])
|
| 48 |
+
validate_duration(duration)
|
| 49 |
+
return duration
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def decode(source: Path, folder: Path, log: Path) -> tuple[Path, Path, float]:
|
| 53 |
+
duration = probe(source)
|
| 54 |
+
pcm48, pcm24 = folder / "source-48k.wav", folder / "source-24k.wav"
|
| 55 |
+
for rate, target in ((48000, pcm48), (24000, pcm24)):
|
| 56 |
+
command(["ffmpeg", "-nostdin", "-v", "error", "-y", "-i", str(source),
|
| 57 |
+
"-vn", "-ac", "1", "-ar", str(rate), "-c:a", "pcm_f32le",
|
| 58 |
+
str(target)], log, timeout=120)
|
| 59 |
+
return pcm48, pcm24, duration
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def source_sha256(path: Path) -> str:
|
| 63 |
+
digest = hashlib.sha256()
|
| 64 |
+
with path.open("rb") as stream:
|
| 65 |
+
for block in iter(lambda: stream.read(1 << 20), b""):
|
| 66 |
+
digest.update(block)
|
| 67 |
+
return digest.hexdigest()
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def transcribe(pcm24: Path, folder: Path, log: Path) -> tuple[Path, Path, Path]:
|
| 71 |
+
command([str(SHEET_PYTHON), str(ROOT / "sheet_worker.py"),
|
| 72 |
+
str(pcm24), str(folder)], log, timeout=900)
|
| 73 |
+
return folder / "notes.mid", folder / "melody.abc", folder / "full.abc"
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def render_instrument(midi_path: Path, folder: Path, instrument: str,
|
| 77 |
+
duration: float, log: Path) -> Path:
|
| 78 |
+
import numpy as np
|
| 79 |
+
import pretty_midi
|
| 80 |
+
import soundfile as sf
|
| 81 |
+
|
| 82 |
+
if instrument not in INSTRUMENTS:
|
| 83 |
+
raise ValueError("Неизвестный инструмент")
|
| 84 |
+
source = pretty_midi.PrettyMIDI(str(midi_path))
|
| 85 |
+
notes = [n for track in source.instruments for n in track.notes]
|
| 86 |
+
if not notes:
|
| 87 |
+
raise RuntimeError("SheetSage2 не нашёл нот в напеве")
|
| 88 |
+
midi = pretty_midi.PrettyMIDI(initial_tempo=120, resolution=10000)
|
| 89 |
+
track = pretty_midi.Instrument(program=INSTRUMENTS[instrument][0], name="Melody")
|
| 90 |
+
for note in notes:
|
| 91 |
+
track.notes.append(pretty_midi.Note(velocity=90, pitch=note.pitch,
|
| 92 |
+
start=note.start, end=note.end))
|
| 93 |
+
midi.instruments.append(track)
|
| 94 |
+
rendered_midi = folder / "instrument.mid"
|
| 95 |
+
midi.write(str(rendered_midi))
|
| 96 |
+
soundfont = Path(pretty_midi.__file__).parent / "TimGM6mb.sf2"
|
| 97 |
+
if not soundfont.is_file():
|
| 98 |
+
raise RuntimeError("SoundFont missing from pretty_midi installation")
|
| 99 |
+
raw = folder / "instrument-tail.wav"
|
| 100 |
+
command(["fluidsynth", "-ni", "-C", "0", "-R", "0", "-r", "48000",
|
| 101 |
+
"-z", "64", "-g", ".2", "-o", "player.timing-source=sample",
|
| 102 |
+
"-o", "synth.min-note-length=0", "-T", "wav", "-O", "float",
|
| 103 |
+
"-F", str(raw), str(soundfont), str(rendered_midi)], log, timeout=120)
|
| 104 |
+
audio, rate = sf.read(raw, dtype="float32", always_2d=True)
|
| 105 |
+
if rate != 48000 or not np.isfinite(audio).all():
|
| 106 |
+
raise RuntimeError("Синтезатор вернул повреждённое аудио")
|
| 107 |
+
frames = round(duration * rate)
|
| 108 |
+
audio = np.pad(audio, ((0, max(0, frames - len(audio))), (0, 0)))[:frames]
|
| 109 |
+
output = folder / "raw.wav"
|
| 110 |
+
sf.write(output, audio, rate, subtype="FLOAT")
|
| 111 |
+
return output
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def render_contour(source: Path, folder: Path) -> Path:
|
| 115 |
+
import librosa
|
| 116 |
+
import numpy as np
|
| 117 |
+
from scipy.ndimage import median_filter
|
| 118 |
+
import soundfile as sf
|
| 119 |
+
|
| 120 |
+
y, rate = sf.read(source, dtype="float32")
|
| 121 |
+
if rate != 48000 or y.ndim != 1:
|
| 122 |
+
raise ValueError("Expected mono 48 kHz input")
|
| 123 |
+
reduced = librosa.resample(y, orig_sr=rate, target_sr=16000)
|
| 124 |
+
f0, voiced, _ = librosa.pyin(reduced, sr=16000, fmin=55., fmax=1000.,
|
| 125 |
+
frame_length=1024, hop_length=160, center=True)
|
| 126 |
+
times = np.arange(len(f0)) * .01
|
| 127 |
+
keep = times < len(y) / rate
|
| 128 |
+
times, f0, voiced = times[keep], f0[keep], voiced[keep]
|
| 129 |
+
squares = np.r_[0., np.cumsum(np.square(y, dtype=np.float64))]
|
| 130 |
+
left = np.maximum(0, np.rint((times - .01) * rate).astype(int))
|
| 131 |
+
right = np.minimum(len(y), np.rint((times + .01) * rate).astype(int))
|
| 132 |
+
rms = np.sqrt((squares[right] - squares[left]) / np.maximum(1, right - left))
|
| 133 |
+
threshold = max(10 ** (-60 / 20), float(np.percentile(rms, 95)) * 10 ** (-35 / 20))
|
| 134 |
+
active = voiced & np.isfinite(f0) & (f0 > 0) & (rms >= threshold)
|
| 135 |
+
boundaries = np.flatnonzero(np.diff(np.r_[False, active, False]))
|
| 136 |
+
wave = np.zeros(len(y), dtype=np.float32)
|
| 137 |
+
for first, last in zip(boundaries[::2], boundaries[1::2]):
|
| 138 |
+
pitch = median_filter(69 + 12 * np.log2(f0[first:last] / 440),
|
| 139 |
+
size=3, mode="nearest")
|
| 140 |
+
start = max(0, round((times[first] - .005) * rate))
|
| 141 |
+
end = min(len(y), round((times[last - 1] + .005) * rate))
|
| 142 |
+
t = np.arange(start, end) / rate
|
| 143 |
+
envelope = np.interp(t, times[first:last], rms[first:last])
|
| 144 |
+
fade = min(round(.005 * rate), len(t) // 2)
|
| 145 |
+
if fade:
|
| 146 |
+
envelope[:fade] *= np.linspace(0, 1, fade)
|
| 147 |
+
envelope[-fade:] *= np.linspace(1, 0, fade)
|
| 148 |
+
frequency = 440 * 2 ** ((np.interp(t, times[first:last], pitch) - 69) / 12)
|
| 149 |
+
phase = 2 * np.pi * np.cumsum(frequency) / rate
|
| 150 |
+
harmonics = np.sin(phase) + .25 * np.sin(2 * phase) + .08 * np.sin(3 * phase)
|
| 151 |
+
wave[start:end] = envelope * harmonics * np.sqrt(2 / (1 + .25**2 + .08**2))
|
| 152 |
+
if not np.any(wave):
|
| 153 |
+
raise RuntimeError("Не удалось выделить звучащую высоту напева")
|
| 154 |
+
output = folder / "raw.wav"
|
| 155 |
+
sf.write(output, wave, rate, subtype="FLOAT")
|
| 156 |
+
return output
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
def render_yue(score_path: Path, folder: Path, instrument: str, mode: str,
|
| 160 |
+
style: str, seed: int, log: Path) -> Path:
|
| 161 |
+
if instrument not in INSTRUMENTS:
|
| 162 |
+
raise ValueError("Неизвестный инструмент")
|
| 163 |
+
if not 0 <= seed < 2**31:
|
| 164 |
+
raise ValueError("Seed должен быть целым числом от 0 до 2³¹−1")
|
| 165 |
+
style = style.strip()
|
| 166 |
+
if not style or len(style) > 500:
|
| 167 |
+
raise ValueError("Промпт YuE2 должен содержать 1–500 символов")
|
| 168 |
+
conditioned = instrumental_abc(score_path.read_text(), keep_chords=mode == "yue_arrangement")
|
| 169 |
+
abc_path = folder / "instrumental.abc"
|
| 170 |
+
abc_path.write_text(conditioned)
|
| 171 |
+
request = {"abc_path": str(abc_path), "style": style,
|
| 172 |
+
"seed": seed, "cot": "full" if mode == "yue_arrangement" else "melody"}
|
| 173 |
+
request_path = folder / "yue-request.json"
|
| 174 |
+
request_path.write_text(json.dumps(request, ensure_ascii=False, indent=2) + "\n")
|
| 175 |
+
command([str(YUE_PYTHON), str(ROOT / "yue_worker.py"),
|
| 176 |
+
str(request_path), str(folder)], log, timeout=1800)
|
| 177 |
+
return folder / "raw.wav"
|
| 178 |
+
|
| 179 |
+
|
| 180 |
+
def encode_mp3(source: Path, destination: Path, log: Path) -> None:
|
| 181 |
+
import numpy as np
|
| 182 |
+
import soundfile as sf
|
| 183 |
+
|
| 184 |
+
audio, rate = sf.read(source, dtype="float32", always_2d=True)
|
| 185 |
+
if not len(audio) or not np.isfinite(audio).all():
|
| 186 |
+
raise RuntimeError("Результат пустой или содержит повреждённые отсчёты")
|
| 187 |
+
peak = float(np.max(np.abs(audio)))
|
| 188 |
+
if peak <= 1e-8:
|
| 189 |
+
raise RuntimeError("Результат слишком тихий")
|
| 190 |
+
gain = min(4., .95 / peak)
|
| 191 |
+
safe = destination.with_suffix(".safe.wav")
|
| 192 |
+
sf.write(safe, audio * gain, rate, subtype="PCM_24")
|
| 193 |
+
command(["ffmpeg", "-nostdin", "-v", "error", "-y", "-i", str(safe),
|
| 194 |
+
"-codec:a", "libmp3lame", "-b:a", "320k", "-ar", "48000",
|
| 195 |
+
str(destination)], log, timeout=120)
|
| 196 |
+
if not destination.is_file() or destination.stat().st_size < 1000:
|
| 197 |
+
raise RuntimeError("MP3 не создан")
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
def generate(source: str, mode_label: str, instrument: str, prompt: str,
|
| 201 |
+
seed: int, progress=None) -> tuple[str, str | None, str | None, dict]:
|
| 202 |
+
if not source or mode_label not in MODES:
|
| 203 |
+
raise ValueError("Загрузите или запишите напев и выберите режим")
|
| 204 |
+
mode = MODES[mode_label]
|
| 205 |
+
folder = Path(tempfile.mkdtemp(prefix="hum-"))
|
| 206 |
+
log = folder / "run.log"
|
| 207 |
+
began = time.monotonic()
|
| 208 |
+
uploaded = Path(source)
|
| 209 |
+
digest = source_sha256(uploaded)
|
| 210 |
+
if progress:
|
| 211 |
+
progress(.02, desc="Проверяю и декодирую напев")
|
| 212 |
+
pcm48, pcm24, duration = decode(uploaded, folder, log)
|
| 213 |
+
midi = abc = None
|
| 214 |
+
if mode == "contour":
|
| 215 |
+
if progress:
|
| 216 |
+
progress(.35, desc="Оцениваю непрерывную высоту")
|
| 217 |
+
raw = render_contour(pcm48, folder)
|
| 218 |
+
else:
|
| 219 |
+
if progress:
|
| 220 |
+
progress(.18, desc="SheetSage2 извлекает ноты и ABC")
|
| 221 |
+
midi, melody_abc, full_abc = transcribe(pcm24, folder, log)
|
| 222 |
+
abc = melody_abc if mode != "yue_arrangement" else full_abc
|
| 223 |
+
if mode == "sheet":
|
| 224 |
+
if progress:
|
| 225 |
+
progress(.7, desc="Играю ноты одним инструментом")
|
| 226 |
+
raw = render_instrument(midi, folder, instrument, duration, log)
|
| 227 |
+
else:
|
| 228 |
+
if progress:
|
| 229 |
+
progress(.55, desc="YuE2 генерирует музыку на GPU")
|
| 230 |
+
raw = render_yue(abc, folder, instrument, mode, prompt, int(seed), log)
|
| 231 |
+
abc = folder / "instrumental.abc"
|
| 232 |
+
if progress:
|
| 233 |
+
progress(.9, desc="Готовлю MP3 320 кбит/с")
|
| 234 |
+
mp3 = folder / "hum-result.mp3"
|
| 235 |
+
encode_mp3(raw, mp3, log)
|
| 236 |
+
metadata = {"mode": mode_label, "instrument": instrument if mode != "contour" else None,
|
| 237 |
+
"source_sha256": digest, "source_seconds": round(duration, 3),
|
| 238 |
+
"result_seconds": round(__import__("soundfile").info(raw).duration, 3),
|
| 239 |
+
"source_timing_preserved": mode in ("sheet", "contour"),
|
| 240 |
+
"vocal_absence_guaranteed": mode in ("sheet", "contour"),
|
| 241 |
+
"elapsed_seconds": round(time.monotonic() - began, 2),
|
| 242 |
+
"notes": "YuE2 may add vocals, instruments, or change timing" if mode.startswith("yue") else None}
|
| 243 |
+
(folder / "result.json").write_text(json.dumps(metadata, ensure_ascii=False, indent=2) + "\n")
|
| 244 |
+
return str(mp3), str(midi) if midi else None, str(abc) if abc else None, metadata
|
sheet_worker.py
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""One isolated SheetSage2 CUDA transcription per request."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import importlib
|
| 6 |
+
import json
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
import shutil
|
| 9 |
+
import sys
|
| 10 |
+
|
| 11 |
+
import numpy as np
|
| 12 |
+
import soundfile as sf
|
| 13 |
+
import torch
|
| 14 |
+
from transformers import AutoModel
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
SHEETSAGE_REV = "eab522a8168e8b8b8c4856bf8609cd86198f01fe"
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
def run(source: Path, output: Path) -> None:
|
| 21 |
+
if not torch.cuda.is_available():
|
| 22 |
+
raise RuntimeError("SheetSage2 requires the configured NVIDIA GPU")
|
| 23 |
+
waveform, rate = sf.read(source, dtype="float32", always_2d=True)
|
| 24 |
+
if rate != 24000 or waveform.shape[1] != 1 or not np.isfinite(waveform).all():
|
| 25 |
+
raise ValueError("Expected finite mono 24 kHz PCM")
|
| 26 |
+
torch.manual_seed(1024)
|
| 27 |
+
np.random.seed(1024)
|
| 28 |
+
model = AutoModel.from_pretrained(
|
| 29 |
+
"m-a-p/SheetSage2", revision=SHEETSAGE_REV, trust_remote_code=True,
|
| 30 |
+
torch_dtype=torch.float32, attn_implementation="sdpa",
|
| 31 |
+
).eval().to("cuda")
|
| 32 |
+
native = output / "native"
|
| 33 |
+
result = model.transcribe(waveform[:, 0], sampling_rate=24000,
|
| 34 |
+
output_dir=native, dtype="fp32", preset="default",
|
| 35 |
+
melody_only=False)
|
| 36 |
+
if result.get("abc_error") or not (native / "score.abc").is_file():
|
| 37 |
+
raise RuntimeError(f"SheetSage2 could not export ABC: {result.get('abc_error')}")
|
| 38 |
+
package = model.__class__.__module__.rsplit(".", 1)[0]
|
| 39 |
+
exporter = importlib.import_module(package + ".exports_sheetsage2").export_result
|
| 40 |
+
events = json.loads((native / "events.json").read_text())
|
| 41 |
+
melody = exporter(events, model.tokenizer, output / "melody-only",
|
| 42 |
+
duration=result["duration_seconds"], melody_only=True)
|
| 43 |
+
if melody.get("abc_error") or not (output / "melody-only/score.abc").is_file():
|
| 44 |
+
raise RuntimeError(f"SheetSage2 could not export melody ABC: {melody.get('abc_error')}")
|
| 45 |
+
shutil.copy2(native / "score.abc", output / "full.abc")
|
| 46 |
+
shutil.copy2(output / "melody-only/score.abc", output / "melody.abc")
|
| 47 |
+
shutil.copy2(native / "melody_vocal.mid", output / "notes.mid")
|
| 48 |
+
(output / "sheet.json").write_text(json.dumps({
|
| 49 |
+
"model": "m-a-p/SheetSage2", "revision": SHEETSAGE_REV,
|
| 50 |
+
"duration_s": result["duration_seconds"],
|
| 51 |
+
"source_timeline": True,
|
| 52 |
+
}, indent=2) + "\n")
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
if __name__ == "__main__":
|
| 56 |
+
run(Path(sys.argv[1]), Path(sys.argv[2]))
|
test_logic.py
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Contracts that prevent vocal ABC leakage and accidental timing expansion."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import unittest
|
| 6 |
+
|
| 7 |
+
from logic import instrumental_abc, validate_duration
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
NATIVE = '''X:1
|
| 11 |
+
M:4/4
|
| 12 |
+
L:1/16
|
| 13 |
+
V: Vocal clef=treble name="Vocal Melody" snm="Vocal"
|
| 14 |
+
V: Ins clef=treble name="Ins Melody" snm="Inst."
|
| 15 |
+
K:B
|
| 16 |
+
% chorus
|
| 17 |
+
V: Vocal
|
| 18 |
+
"Bm"z12F,4|
|
| 19 |
+
V: Ins
|
| 20 |
+
Z1|
|
| 21 |
+
V: Vocal
|
| 22 |
+
Z|
|
| 23 |
+
V: Ins
|
| 24 |
+
Z1|
|
| 25 |
+
'''
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
class LogicTest(unittest.TestCase):
|
| 29 |
+
def test_move_melody_to_instrument_without_chord_or_voice(self):
|
| 30 |
+
result = instrumental_abc(NATIVE)
|
| 31 |
+
self.assertNotIn("Vocal", result)
|
| 32 |
+
self.assertNotIn('"Bm"', result)
|
| 33 |
+
self.assertIn("z12F,4|", result)
|
| 34 |
+
self.assertIn("V: Ins\nz12F,4|", result)
|
| 35 |
+
self.assertIn("V: Ins\nZ|", result)
|
| 36 |
+
self.assertLess(result.index("V: Ins clef"), result.index("K:B"))
|
| 37 |
+
|
| 38 |
+
def test_composition_keeps_chords_but_no_vocal_slot(self):
|
| 39 |
+
result = instrumental_abc(NATIVE, keep_chords=True)
|
| 40 |
+
self.assertIn('"Bm"z12F,4|', result)
|
| 41 |
+
self.assertNotIn("Vocal", result)
|
| 42 |
+
|
| 43 |
+
def test_unknown_layout_fails(self):
|
| 44 |
+
with self.assertRaises(ValueError):
|
| 45 |
+
instrumental_abc(NATIVE.replace("Z1|", "z16|", 1))
|
| 46 |
+
|
| 47 |
+
def test_duration_bounds(self):
|
| 48 |
+
validate_duration(0.4)
|
| 49 |
+
validate_duration(35)
|
| 50 |
+
for seconds in (0.1, 35.1):
|
| 51 |
+
with self.assertRaises(ValueError):
|
| 52 |
+
validate_duration(seconds)
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
if __name__ == "__main__":
|
| 56 |
+
unittest.main()
|
yue_worker.py
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""One isolated YuE2 CUDA generation from the reviewed SheetSage ABC form."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import json
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
import sys
|
| 8 |
+
|
| 9 |
+
import numpy as np
|
| 10 |
+
import soundfile as sf
|
| 11 |
+
import torch
|
| 12 |
+
from yue2 import YuE2Pipeline
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
YUE_REV = "29b3558dd46954a0cd9021dc76d5c91864a0f1c7"
|
| 16 |
+
VAE_REV = "9a94e1d0ea9f8087e98f77fa88df4a4068104d2a"
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
def run(request_path: Path, output: Path) -> None:
|
| 20 |
+
if not torch.cuda.is_available() or not torch.cuda.is_bf16_supported():
|
| 21 |
+
raise RuntimeError("YuE2 requires a BF16-capable NVIDIA GPU")
|
| 22 |
+
request = json.loads(request_path.read_text())
|
| 23 |
+
score = Path(request["abc_path"]).read_text()
|
| 24 |
+
pipe = YuE2Pipeline.from_pretrained(
|
| 25 |
+
"m-a-p/YuE2-3B", vae="m-a-p/YuE2-Vae",
|
| 26 |
+
revision=YUE_REV, vae_revision=VAE_REV,
|
| 27 |
+
device="cuda", backend="torch-eager", quantization="none",
|
| 28 |
+
offload_ar=False, memory_budget_gib=44, progress=True,
|
| 29 |
+
)
|
| 30 |
+
try:
|
| 31 |
+
song = pipe(style=request["style"], lyrics="", cot=request["cot"],
|
| 32 |
+
abc=score, seed=request["seed"], cfg_scale=1.0,
|
| 33 |
+
semantic_sampling={"max_tokens": 1100})
|
| 34 |
+
audio = np.asarray(song.audio, dtype=np.float32)
|
| 35 |
+
if audio.ndim != 2 or audio.shape[1] != 2 or not np.isfinite(audio).all():
|
| 36 |
+
raise RuntimeError("YuE2 returned invalid stereo audio")
|
| 37 |
+
if not len(audio) or float(np.max(np.abs(audio))) == 0:
|
| 38 |
+
raise RuntimeError("YuE2 returned silence")
|
| 39 |
+
sf.write(output / "raw.wav", audio, song.sample_rate, subtype="FLOAT")
|
| 40 |
+
(output / "yue.json").write_text(json.dumps({
|
| 41 |
+
"model": "m-a-p/YuE2-3B", "revision": YUE_REV,
|
| 42 |
+
"vae_revision": VAE_REV, "duration_s": len(audio) / song.sample_rate,
|
| 43 |
+
"truncated": song.truncated, "timing": song.timing,
|
| 44 |
+
"source_timeline": False, "vocal_absence_guaranteed": False,
|
| 45 |
+
}, indent=2) + "\n")
|
| 46 |
+
finally:
|
| 47 |
+
pipe.close()
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
if __name__ == "__main__":
|
| 51 |
+
run(Path(sys.argv[1]), Path(sys.argv[2]))
|