"use client"; import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { LyricsEditor } from "@/components/lyrics-editor"; import { Queue } from "@/components/queue"; import { EMPTY_REFERENCE, ReferenceDrop, type ReferenceState } from "@/components/reference-drop"; import { RenderPanel, type RenderSettings } from "@/components/render-panel"; import { ScorePanel } from "@/components/score-panel"; import { StyleField } from "@/components/style-field"; import { Transport } from "@/components/transport"; import { IDLE_LIFT, useLyricLift } from "@/lib/lyrics-lift"; import { DEFAULT_ADVANCED, STARTER_LYRICS, STARTER_STYLE } from "@/lib/presets"; import type { ComfyCapabilities, Job } from "@/lib/types"; /** * The SSE stream emits a job the moment the store creates it, which can beat * the POST response that also carries it. Merging by id keeps one row per job * whichever arrives first, and keeps the newer snapshot of the two. */ function upsertJob(jobs: Job[], incoming: Job): Job[] { const existing = jobs.find((job) => job.id === incoming.id); const merged = existing ? { ...existing, ...incoming } : incoming; return [merged, ...jobs.filter((job) => job.id !== incoming.id)].sort( (a, b) => b.createdAt - a.createdAt, ); } const INITIAL_SETTINGS: RenderSettings = { scoreSource: "model", mode: "full", maxDuration: 240, seed: 0, randomizeSeed: true, tiledDecode: false, realAudioMatch: false, realAudioStrength: 1, instrumentalArrangement: false, instrumentalStrength: 1, outputFormat: "flac", checkpoint: "", audioEncoder: "", advanced: DEFAULT_ADVANCED, }; export function Studio() { const [style, setStyle] = useState(STARTER_STYLE); const [lyrics, setLyrics] = useState(STARTER_LYRICS); const [reference, setReference] = useState(EMPTY_REFERENCE); const [settings, setSettings] = useState(INITIAL_SETTINGS); const [capabilities, setCapabilities] = useState(null); const [jobs, setJobs] = useState([]); const [selectedId, setSelectedId] = useState(null); const [manualScore, setManualScore] = useState(""); const { lift, start: startLift, reset: resetLift, setLift } = useLyricLift(); const [scoreOpen, setScoreOpen] = useState(false); const [playToken, setPlayToken] = useState(0); const [busy, setBusy] = useState(false); const [error, setError] = useState(null); const autoSelected = useRef(null); // Safari will not decode FLAC, so do not hand it a FLAC by default. useEffect(() => { const probe = document.createElement("audio"); if (!probe.canPlayType("audio/flac") && !probe.canPlayType("audio/x-flac")) { setSettings((current) => ({ ...current, outputFormat: "mp3" })); } }, []); // Keep looking until ComfyUI answers, so starting it later just works. useEffect(() => { let cancelled = false; let timer: ReturnType | null = null; const poll = () => { void fetch("/api/capabilities") .then((res) => res.json()) .then((data: ComfyCapabilities) => { if (cancelled) return; setCapabilities(data); setSettings((current) => ({ ...current, checkpoint: current.checkpoint || pickWeight(data.checkpoints, /yue2/i), audioEncoder: current.audioEncoder || pickWeight(data.audioEncoders, /sheetsage/i), })); if (!data.connected) timer = setTimeout(poll, 8000); }) .catch(() => { if (!cancelled) timer = setTimeout(poll, 8000); }); }; poll(); return () => { cancelled = true; if (timer) clearTimeout(timer); }; }, []); useEffect(() => { const source = new EventSource("/api/events"); source.addEventListener("snapshot", (event) => { setJobs(JSON.parse((event as MessageEvent).data) as Job[]); }); source.addEventListener("job", (event) => { setJobs((current) => upsertJob(current, JSON.parse((event as MessageEvent).data) as Job)); }); return () => source.close(); }, []); // The newest finished take takes over the transport as soon as it lands. useEffect(() => { const done = jobs.find((job) => job.stage === "done"); if (done && autoSelected.current !== done.id) { autoSelected.current = done.id; setSelectedId(done.id); } }, [jobs]); const selected = useMemo( () => jobs.find((job) => job.id === selectedId) ?? null, [jobs, selectedId], ); const running = useMemo( () => jobs.find((job) => !["done", "failed", "cancelled"].includes(job.stage)) ?? null, [jobs], ); // An edited score wins over whatever the selected take came back with. const activeScore = manualScore || selected?.abc || ""; // Dropping a reference switches the score source, because that is what the // drop was for. It used to force the plan to melody as well, which quietly // undid a deliberate "melody and chords" choice, and that is the setting // that decides how much of the reference survives. The plan is left alone. const handleReference = useCallback( (next: ReferenceState) => { setReference(next); setSettings((current) => ({ ...current, scoreSource: next.status === "ready" ? "reference" : current.scoreSource === "reference" ? "model" : current.scoreSource, })); if (next.status === "ready" && next.filename) { void startLift(next.filename, next.start, next.duration); } else if (next.status !== "uploading") { resetLift(); } }, [startLift, resetLift], ); // Filling the field is the point, but only when there is nothing to lose. // Untouched starter text is not the operator's work; anything they typed is, // and that only ever gets replaced by someone pressing the button. const lyricsAreDisposable = !lyrics.trim() || lyrics === STARTER_LYRICS; useEffect(() => { if (lift.status !== "done" || !lift.lyrics) return; if (!lyricsAreDisposable) return; setLyrics(lift.lyrics); setLift((current) => ({ ...current, lyrics: null })); }, [lift.status, lift.lyrics, lyricsAreDisposable, setLift]); const applyLift = useCallback(() => { if (!lift.lyrics) return; setLyrics(lift.lyrics); setLift(IDLE_LIFT); }, [lift.lyrics, setLift]); const retryLift = useCallback(() => { if (reference.status !== "ready" || !reference.filename) return; void startLift(reference.filename, reference.start, reference.duration); }, [reference.status, reference.filename, reference.start, reference.duration, startLift]); const render = useCallback(async () => { setBusy(true); setError(null); try { const res = await fetch("/api/generate", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ style, lyrics, mode: settings.mode, scoreSource: settings.scoreSource, abc: settings.scoreSource === "manual" ? activeScore : "", referenceFilename: reference.filename, referenceStart: reference.start, referenceDuration: reference.duration, maxDuration: settings.maxDuration, seed: settings.seed, randomizeSeed: settings.randomizeSeed, checkpoint: settings.checkpoint, audioEncoder: settings.audioEncoder, tiledDecode: settings.tiledDecode, realAudioMatch: settings.realAudioMatch, realAudioStrength: settings.realAudioStrength, instrumentalArrangement: settings.instrumentalArrangement, instrumentalStrength: settings.instrumentalStrength, outputFormat: settings.outputFormat, advanced: settings.advanced, }), }); const body = await res.json(); if (!res.ok) throw new Error(body.error ?? "The render could not start"); setJobs((current) => upsertJob(current, body as Job)); } catch (caught) { setError(caught instanceof Error ? caught.message : "The render could not start"); } finally { setBusy(false); } }, [style, lyrics, settings, reference.filename, reference.start, reference.duration, activeScore]); const cancel = useCallback(async (job: Job) => { await fetch(`/api/jobs/${job.id}`, { method: "DELETE" }).catch(() => undefined); }, []); return (
setLift(IDLE_LIFT)} onRetryLift={retryLift} />
{/* Deliberately not a sticky, independently scrolling column. It used to be, sized against a hardcoded header height, and the moment the error banner expanded the header grew past that number: the column's bottom ran off screen and Takes could only be reached through a thin inner scrollbar. One page scroll cannot trap anything. */}
setScoreOpen(true)} busy={busy} error={error} onRender={render} /> { setSelectedId(job.id); setManualScore(""); setPlayToken((token) => token + 1); }} onCancel={cancel} />
setScoreOpen(true)} /> setScoreOpen(false)} onApply={(next) => { setManualScore(next); setSettings((current) => ({ ...current, scoreSource: "manual" })); }} />
); } /** Prefer the weight that looks like the one we want over whatever sorts first. */ function pickWeight(options: string[], wanted: RegExp): string { return options.find((name) => wanted.test(name)) ?? options[0] ?? ""; } function Header({ capabilities }: { capabilities: ComfyCapabilities | null }) { const connected = capabilities?.connected ?? false; const missing = capabilities?.missingNodes ?? []; return ( <> {/* Only the bar is sticky. The notices below it are diagnostic, and an expanded engine log is tall enough that pinning it would eat a third of a laptop viewport and push the right column out of reach. */}
YuE2 Studio Full songs from a style, a lyric sheet and a reference
{/* Real semantic state, the only status indicator on the page. */} {capabilities === null ? "checking" : capabilities.mock ? "mock backend" : connected ? capabilities.url.replace(/^https?:\/\//, "") : "ComfyUI unreachable"} {capabilities?.buildStamp ? ( {capabilities.buildStamp} ) : null}
{capabilities && !capabilities.connected ? (

Cannot reach ComfyUI at {capabilities.url}. It may still be loading the model, and this page picks it up on its own once it answers.

{capabilities.engineLog ? (
Engine log
                {capabilities.engineLog}
              
) : null}
) : null} {missing.length > 0 ? (

This ComfyUI build is missing {missing.slice(0, 3).join(", ")}. Update to v0.35.0 or newer.

) : null} ); }