salgadev commited on
Commit
44948ab
Β·
verified Β·
1 Parent(s): 5f6cd09

Sync from GitHub af6d0631

Browse files
README.md CHANGED
@@ -15,6 +15,8 @@ short_description: You start playing, it joins in β€” an AI music companion.
15
  > tempo, in the instrument you pick. Built for the Stability AI Challenge at
16
  > Music Hackspace Montreal (August 22–23, 2026).
17
 
 
 
18
  ## What this is
19
 
20
  **Jam Buddy** is a call-and-response music practice partner. You start playing,
 
15
  > tempo, in the instrument you pick. Built for the Stability AI Challenge at
16
  > Music Hackspace Montreal (August 22–23, 2026).
17
 
18
+ TRY IT NOW AT [Our HuggingFace Space](https://huggingface.co/spaces/salgadev/jam-buddy)
19
+
20
  ## What this is
21
 
22
  **Jam Buddy** is a call-and-response music practice partner. You start playing,
apps/web/__tests__/visualizer.test.tsx ADDED
@@ -0,0 +1,103 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
2
+ import { render, screen, cleanup, act } from "@testing-library/react";
3
+ import userEvent from "@testing-library/user-event";
4
+ import { createRef } from "react";
5
+ import { Visualizer, type VisualizerHandle } from "@/lib/jambuddy/visualizer";
6
+
7
+ /**
8
+ * Regression test for the "one playback at a time" rule.
9
+ *
10
+ * Each playable waveform owns its own <Audio>. The parent wires onStartPlayback
11
+ * to stop any other playback before a new one starts. This pins:
12
+ * - clicking a playable waveform's toggle fires onStartPlayback FIRST
13
+ * - the exposed stop() handle actually stops the audio and resets the toggle
14
+ * So PLAY TOGETHER / another waveform can always silence a playing one.
15
+ */
16
+
17
+ // Fake Audio so `new Audio(url)` works under happy-dom.
18
+ class FakeAudio {
19
+ onended: (() => void) | null = null;
20
+ paused = true;
21
+ played = false;
22
+ play() {
23
+ this.played = true;
24
+ this.paused = false;
25
+ return Promise.resolve();
26
+ }
27
+ pause() {
28
+ this.paused = true;
29
+ }
30
+ }
31
+ let fakeAudio: FakeAudio | null = null;
32
+
33
+ // Fake AudioContext + fetch so the waveform-draw effect completes cleanly.
34
+ class FakeAudioContext {
35
+ sampleRate = 44100;
36
+ close() {
37
+ return Promise.resolve();
38
+ }
39
+ decodeAudioData() {
40
+ return Promise.resolve({
41
+ numberOfChannels: 1,
42
+ sampleRate: 44100,
43
+ getChannelData: () => new Float32Array(100),
44
+ });
45
+ }
46
+ }
47
+
48
+ beforeEach(() => {
49
+ fakeAudio = new FakeAudio();
50
+ vi.stubGlobal("Audio", class { constructor() { return fakeAudio; } });
51
+ vi.stubGlobal("AudioContext", FakeAudioContext);
52
+ vi.stubGlobal("fetch", vi.fn(() =>
53
+ Promise.resolve({ arrayBuffer: () => Promise.resolve(new ArrayBuffer(0)) }),
54
+ ));
55
+ });
56
+
57
+ afterEach(() => {
58
+ cleanup();
59
+ vi.unstubAllGlobals();
60
+ });
61
+
62
+ describe("Visualizer playable waveform", () => {
63
+ it("fires onStartPlayback BEFORE starting play (exclusivity guard)", async () => {
64
+ const user = userEvent.setup();
65
+ const onStart = vi.fn();
66
+ const ref = createRef<VisualizerHandle>();
67
+ await act(async () => {
68
+ render(
69
+ <Visualizer
70
+ ref={ref}
71
+ audioUrl="blob:take"
72
+ playable
73
+ onStartPlayback={onStart}
74
+ />,
75
+ );
76
+ });
77
+
78
+ await user.click(screen.getByRole("button", { name: "Play" }));
79
+
80
+ // The owner must be told to stop anything else before this one starts.
81
+ expect(onStart).toHaveBeenCalledTimes(1);
82
+ expect(fakeAudio?.played).toBe(true);
83
+ });
84
+
85
+ it("exposes a stop() handle that stops audio and flips the toggle back", async () => {
86
+ const user = userEvent.setup();
87
+ const ref = createRef<VisualizerHandle>();
88
+ await act(async () => {
89
+ render(<Visualizer ref={ref} audioUrl="blob:take" playable />);
90
+ });
91
+
92
+ await user.click(screen.getByRole("button", { name: "Play" }));
93
+ // Now playing -> the toggle reads Stop.
94
+ expect(screen.getByRole("button", { name: "Stop playback" })).toBeInTheDocument();
95
+
96
+ act(() => {
97
+ ref.current?.stop();
98
+ });
99
+ expect(fakeAudio?.paused).toBe(true);
100
+ // stop() resets internal state, so the toggle reads Play again.
101
+ expect(screen.getByRole("button", { name: "Play" })).toBeInTheDocument();
102
+ });
103
+ });
apps/web/app/api/jambuddy/route.ts CHANGED
@@ -17,6 +17,9 @@ interface JambuddyBody {
17
  midi?: string;
18
  /** Base64-encoded audio take (mic/interface/render) for audio-to-audio. */
19
  audio?: string;
 
 
 
20
  /** Response length in seconds. Default 30 (ignored when a take is provided). */
21
  duration?: number;
22
  /** Generation backend: "local" (CPU SA3) or "api" (Stable Audio 3.0 Large, 26 credits/gen). */
@@ -166,7 +169,10 @@ export async function POST(req: NextRequest) {
166
  } else if (body.audio) {
167
  // Audio: pass to SA3 via init_audio so it responds to the groove. Tempo
168
  // is still the knob; the take sets duration + drives audio-to-audio.
169
- const audioPath = join(tmpdir(), `jambuddy-take-${Date.now()}.wav`);
 
 
 
170
  await writeFile(audioPath, Buffer.from(body.audio, "base64"));
171
  args.push("--wav", audioPath);
172
  args.push("--genre", body.knobs.genre);
 
17
  midi?: string;
18
  /** Base64-encoded audio take (mic/interface/render) for audio-to-audio. */
19
  audio?: string;
20
+ /** Real extension of the audio take (aif/wav/mp3/...) β€” keeps the temp file
21
+ * name matching its content so soundfile can read it. */
22
+ audioExt?: string;
23
  /** Response length in seconds. Default 30 (ignored when a take is provided). */
24
  duration?: number;
25
  /** Generation backend: "local" (CPU SA3) or "api" (Stable Audio 3.0 Large, 26 credits/gen). */
 
169
  } else if (body.audio) {
170
  // Audio: pass to SA3 via init_audio so it responds to the groove. Tempo
171
  // is still the knob; the take sets duration + drives audio-to-audio.
172
+ // Use the real audio extension (AIFF/WAV/MP3/...) so soundfile can read
173
+ // the temp file β€” a .wav-named AIFF/WEBM fails "Format not recognised".
174
+ const ext = body.audioExt?.match(/^[a-z0-9]{1,4}$/) ? body.audioExt : "wav";
175
+ const audioPath = join(tmpdir(), `jambuddy-take-${Date.now()}.${ext}`);
176
  await writeFile(audioPath, Buffer.from(body.audio, "base64"));
177
  args.push("--wav", audioPath);
178
  args.push("--genre", body.knobs.genre);
apps/web/app/page.tsx CHANGED
@@ -29,7 +29,7 @@ import {
29
  type AudioInput,
30
  type MidiInput,
31
  } from "@/lib/jambuddy/recorder";
32
- import { Visualizer } from "@/lib/jambuddy/visualizer";
33
 
34
  /**
35
  * Jam Buddy β€” "you start playing, it joins in."
@@ -158,6 +158,15 @@ function Pad({
158
  );
159
  }
160
 
 
 
 
 
 
 
 
 
 
161
  export default function HomePage() {
162
  const [instrument, setInstrument] = useState<BuddyInstrument>("bass");
163
  const [inputInstrument, setInputInstrument] = useState<InputInstrument>("other");
@@ -175,6 +184,10 @@ export default function HomePage() {
175
  const [usedBpm, setUsedBpm] = useState<number | null>(null);
176
  const [usedSeconds, setUsedSeconds] = useState<number | null>(null);
177
  const [busy, setBusy] = useState(false);
 
 
 
 
178
  // Generation backend. API (Stable Audio 3.0 Large) is default when the key is
179
  // present β€” fast + better isolation, 26 credits/gen. Local = CPU small model,
180
  // free, supports the negative prompt, slower.
@@ -230,8 +243,38 @@ export default function HomePage() {
230
  );
231
  }
232
 
233
- /** Handle a single uploaded take (MIDI or audio), auto-detecting the type. */
234
- async function handleTakeFile(f: File | null) {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
235
  // One upload slot for either kind; picking a new file clears the old take.
236
  setMidiFile(null);
237
  setMidiBytes(null);
@@ -256,7 +299,11 @@ export default function HomePage() {
256
  }
257
  // Pre-fill the tempo knob from the take's tempo map (Option A: detect
258
  // first, knob stays authoritative + editable).
259
- await prefillTempo(f, "midi");
 
 
 
 
260
  setStatus(
261
  `Loaded ${f.name}. Tempo auto-detected (${bpm} BPM); adjust the knob if needed.`,
262
  );
@@ -266,7 +313,11 @@ export default function HomePage() {
266
  // Audio is heard by the buddy (audio-to-audio). Clear the MIDI-driven
267
  // input-instrument default so the user's declaration reflects the audio.
268
  setInputInstrument("other");
269
- await prefillTempo(f, "audio");
 
 
 
 
270
  setStatus(
271
  `Loaded ${f.name}. Tempo auto-detected (${bpm} BPM); adjust the knob if needed.`,
272
  );
@@ -435,7 +486,17 @@ export default function HomePage() {
435
  * click can read a stale `false` and start a SECOND simultaneous layer. Keep
436
  * a synchronous ref so the toggle is atomic. */
437
  const playbackRef = useRef<{ stop: () => void } | null>(null);
438
- const audioRef = useRef<HTMLAudioElement | null>(null);
 
 
 
 
 
 
 
 
 
 
439
  async function playBoth() {
440
  if (playbackRef.current) {
441
  // Stop: kill current playback (synchronous β€” immune to stale state).
@@ -449,8 +510,9 @@ export default function HomePage() {
449
  setStatus("Generate a response first.");
450
  return;
451
  }
452
- // Pause the standalone audio element so the response isn't heard twice.
453
- audioRef.current?.pause();
 
454
  playbackRef.current = { stop: () => {} }; // claim the toggle synchronously
455
  setIsPlayingTogether(true);
456
  setStatus("Playing your take + the buddy together…");
@@ -504,6 +566,9 @@ export default function HomePage() {
504
  bpm?: number;
505
  midi?: string;
506
  audio?: string;
 
 
 
507
  duration?: number;
508
  mode: "api" | "local";
509
  } = { knobs: { instrument, inputInstrument, genre, mood, bpm }, mode };
@@ -520,6 +585,11 @@ export default function HomePage() {
520
  setStatus("Reading your audio take…");
521
  const base64 = await fileToBase64(audioFile);
522
  payload.audio = base64;
 
 
 
 
 
523
  // Audio-to-audio: the buddy responds to the groove.
524
  delete payload.bpm;
525
  }
@@ -676,6 +746,29 @@ export default function HomePage() {
676
  {audioFile.name} β€” audio: buddy responds to its groove (audio-to-audio).
677
  </p>
678
  )}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
679
  </section>
680
 
681
  {/* Transport */}
@@ -863,16 +956,6 @@ export default function HomePage() {
863
  Generated in {usedSeconds.toFixed(1)}s
864
  </p>
865
  )}
866
- {audioUrl && (
867
- <audio
868
- ref={audioRef}
869
- controls
870
- src={audioUrl}
871
- className="mt-3 w-full"
872
- >
873
- Your browser does not support audio playback.
874
- </audio>
875
- )}
876
  </section>
877
 
878
  {/* Take + response visualizers β€” stacked vertically, DAW-style.
@@ -911,8 +994,11 @@ export default function HomePage() {
911
  <div className="flex items-end gap-3">
912
  <div className="flex-1">
913
  <Visualizer
 
914
  audioUrl={takeAudioUrl}
915
  label="Your take (audio waveform)"
 
 
916
  />
917
  </div>
918
  </div>
@@ -921,8 +1007,11 @@ export default function HomePage() {
921
  <div className="flex items-end gap-3">
922
  <div className="flex-1">
923
  <Visualizer
 
924
  audioUrl={audioUrl}
925
  label="Buddy response (waveform)"
 
 
926
  />
927
  </div>
928
  <button
 
29
  type AudioInput,
30
  type MidiInput,
31
  } from "@/lib/jambuddy/recorder";
32
+ import { Visualizer, type VisualizerHandle } from "@/lib/jambuddy/visualizer";
33
 
34
  /**
35
  * Jam Buddy β€” "you start playing, it joins in."
 
158
  );
159
  }
160
 
161
+ /** Bundled demo takes (Gradio-style clickable examples).
162
+ * Each is an MP3 (loads as an audio take so the buddy responds to it) with a
163
+ * matching BPM (from the source MIDI) that pre-sets the tempo knob. */
164
+ const DEMO_MIDIS = [
165
+ { name: "tupatutupatututata", label: "Tupatutupatututata (drums)", bpm: 158 },
166
+ { name: "dangerous-bass-line", label: "Dangerous bass line", bpm: 120 },
167
+ { name: "this-riff-does-not-exist", label: "This riff does not exist", bpm: 160 },
168
+ ];
169
+
170
  export default function HomePage() {
171
  const [instrument, setInstrument] = useState<BuddyInstrument>("bass");
172
  const [inputInstrument, setInputInstrument] = useState<InputInstrument>("other");
 
184
  const [usedBpm, setUsedBpm] = useState<number | null>(null);
185
  const [usedSeconds, setUsedSeconds] = useState<number | null>(null);
186
  const [busy, setBusy] = useState(false);
187
+ // Name of the demo chip currently loaded as the take (for active highlight).
188
+ const [loadedDemo, setLoadedDemo] = useState<string | null>(null);
189
+ // Monotonic token so a slow demo fetch can't clobber a newer selection.
190
+ const demoLoadToken = useRef(0);
191
  // Generation backend. API (Stable Audio 3.0 Large) is default when the key is
192
  // present β€” fast + better isolation, 26 credits/gen. Local = CPU small model,
193
  // free, supports the negative prompt, slower.
 
243
  );
244
  }
245
 
246
+ /** Load a bundled demo as an audio take + set the tempo knob to its BPM.
247
+ * The MP3 is what the buddy responds to (audio-to-audio); the exact tempo
248
+ * comes from the source MIDI, so we set the knob to it rather than trusting
249
+ * librosa's estimate of the MP3. */
250
+ async function loadDemo(name: string, demoBpm: number) {
251
+ // Select immediately (single-select) + pin the knob, so the UI responds
252
+ // instantly instead of waiting on the slow fetch/detect below.
253
+ const token = ++demoLoadToken.current;
254
+ setLoadedDemo(name);
255
+ setBpm(demoBpm);
256
+ setStatus(`Loading demo "${name}"…`);
257
+ try {
258
+ const res = await fetch(`/demos/${name}.mp3`);
259
+ if (!res.ok) throw new Error(`fetch ${name}.mp3 -> ${res.status}`);
260
+ const blob = await res.blob();
261
+ const file = new File([blob], `${name}.mp3`, { type: "audio/mpeg" });
262
+ // Ignore a stale load if the user has since picked a different demo.
263
+ if (token !== demoLoadToken.current) return;
264
+ // Load it as an audio take; knownBpm skips the slow server detect.
265
+ await handleTakeFile(file, demoBpm);
266
+ setStatus(`Loaded demo "${name}" @ ${demoBpm} BPM.`);
267
+ } catch (e) {
268
+ if (token === demoLoadToken.current) {
269
+ setStatus(`Couldn't load demo: ${String(e)}`);
270
+ }
271
+ }
272
+ }
273
+
274
+ /** Handle a single uploaded take (MIDI or audio), auto-detecting the type.
275
+ * `knownBpm` (optional) skips the slow server detect and pins the knob to a
276
+ * caller-supplied tempo β€” used by demo chips, which already know their BPM. */
277
+ async function handleTakeFile(f: File | null, knownBpm?: number) {
278
  // One upload slot for either kind; picking a new file clears the old take.
279
  setMidiFile(null);
280
  setMidiBytes(null);
 
299
  }
300
  // Pre-fill the tempo knob from the take's tempo map (Option A: detect
301
  // first, knob stays authoritative + editable).
302
+ if (knownBpm !== undefined) {
303
+ setBpm(knownBpm);
304
+ } else {
305
+ await prefillTempo(f, "midi");
306
+ }
307
  setStatus(
308
  `Loaded ${f.name}. Tempo auto-detected (${bpm} BPM); adjust the knob if needed.`,
309
  );
 
313
  // Audio is heard by the buddy (audio-to-audio). Clear the MIDI-driven
314
  // input-instrument default so the user's declaration reflects the audio.
315
  setInputInstrument("other");
316
+ if (knownBpm !== undefined) {
317
+ setBpm(knownBpm);
318
+ } else {
319
+ await prefillTempo(f, "audio");
320
+ }
321
  setStatus(
322
  `Loaded ${f.name}. Tempo auto-detected (${bpm} BPM); adjust the knob if needed.`,
323
  );
 
486
  * click can read a stale `false` and start a SECOND simultaneous layer. Keep
487
  * a synchronous ref so the toggle is atomic. */
488
  const playbackRef = useRef<{ stop: () => void } | null>(null);
489
+ // Refs to the take + response visualizers so PLAY TOGETHER / other toggles
490
+ // can stop their audio (enforces one playback at a time).
491
+ const takeVisualizerRef = useRef<VisualizerHandle>(null);
492
+ const responseVisualizerRef = useRef<VisualizerHandle>(null);
493
+
494
+ // Central authority: stop every waveform toggle before any new playback
495
+ // starts, so the same audio never plays in two places at once.
496
+ const stopAllWaveformPlayback = () => {
497
+ takeVisualizerRef.current?.stop();
498
+ responseVisualizerRef.current?.stop();
499
+ };
500
  async function playBoth() {
501
  if (playbackRef.current) {
502
  // Stop: kill current playback (synchronous β€” immune to stale state).
 
510
  setStatus("Generate a response first.");
511
  return;
512
  }
513
+ // Stop any single-waveform playback so the same audio isn't heard twice
514
+ // while PLAY TOGETHER runs.
515
+ stopAllWaveformPlayback();
516
  playbackRef.current = { stop: () => {} }; // claim the toggle synchronously
517
  setIsPlayingTogether(true);
518
  setStatus("Playing your take + the buddy together…");
 
566
  bpm?: number;
567
  midi?: string;
568
  audio?: string;
569
+ /** Real extension of the audio take (e.g. aif, wav, mp3) so the server
570
+ * names the temp file correctly. */
571
+ audioExt?: string;
572
  duration?: number;
573
  mode: "api" | "local";
574
  } = { knobs: { instrument, inputInstrument, genre, mood, bpm }, mode };
 
585
  setStatus("Reading your audio take…");
586
  const base64 = await fileToBase64(audioFile);
587
  payload.audio = base64;
588
+ // Pass the real audio extension so the server names the temp file
589
+ // correctly (soundfile won't read a .wav-named AIFF/WEBM).
590
+ const extMatch = audioFile.name.match(/\.(aiff?|wav|mp3|flac|ogg|m4a|webm)$/i);
591
+ const ext = extMatch?.[1] ?? "wav";
592
+ payload.audioExt = ext.toLowerCase();
593
  // Audio-to-audio: the buddy responds to the groove.
594
  delete payload.bpm;
595
  }
 
746
  {audioFile.name} β€” audio: buddy responds to its groove (audio-to-audio).
747
  </p>
748
  )}
749
+ {/* Gradio-style demo examples: click to load as a take. Playback is
750
+ on the take waveform (audio) below. */}
751
+ <div className="mt-3 flex flex-wrap items-center gap-2">
752
+ <span className="font-mono text-[10px] uppercase tracking-widest text-[#7f829c]">
753
+ Demos
754
+ </span>
755
+ {DEMO_MIDIS.map((d) => (
756
+ <button
757
+ key={d.name}
758
+ type="button"
759
+ onClick={() => loadDemo(d.name, d.bpm)}
760
+ disabled={busy}
761
+ aria-pressed={loadedDemo === d.name}
762
+ className={`rounded-full border px-3 py-1 text-xs ${
763
+ loadedDemo === d.name
764
+ ? "border-[#5fd38a] bg-[#5fd38a]/15 text-[#5fd38a]"
765
+ : "border-[#2a2d3d] bg-[#1a1c28] text-[#e8e8f0] hover:border-[#5fd38a] hover:text-[#5fd38a]"
766
+ }`}
767
+ >
768
+ {d.label}
769
+ </button>
770
+ ))}
771
+ </div>
772
  </section>
773
 
774
  {/* Transport */}
 
956
  Generated in {usedSeconds.toFixed(1)}s
957
  </p>
958
  )}
 
 
 
 
 
 
 
 
 
 
959
  </section>
960
 
961
  {/* Take + response visualizers β€” stacked vertically, DAW-style.
 
994
  <div className="flex items-end gap-3">
995
  <div className="flex-1">
996
  <Visualizer
997
+ ref={takeVisualizerRef}
998
  audioUrl={takeAudioUrl}
999
  label="Your take (audio waveform)"
1000
+ playable
1001
+ onStartPlayback={stopAllWaveformPlayback}
1002
  />
1003
  </div>
1004
  </div>
 
1007
  <div className="flex items-end gap-3">
1008
  <div className="flex-1">
1009
  <Visualizer
1010
+ ref={responseVisualizerRef}
1011
  audioUrl={audioUrl}
1012
  label="Buddy response (waveform)"
1013
+ playable
1014
+ onStartPlayback={stopAllWaveformPlayback}
1015
  />
1016
  </div>
1017
  <button
apps/web/lib/jambuddy/player.ts CHANGED
@@ -188,6 +188,54 @@ export function midiTempo(buf: ArrayBuffer): number {
188
  }
189
  }
190
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
191
  /**
192
  * Get the total duration (seconds) of a MIDI file β€” the time of the last note
193
  * end. This is what the generated response must match so the take and the
 
188
  }
189
  }
190
 
191
+ /**
192
+ * Play a MIDI take SOLO through the synth (no buddy). Used by the demo
193
+ * examples so a MIDI can be previewed on its own.
194
+ *
195
+ * @param midiBytes the MIDI bytes to play
196
+ * @returns a stop() handle
197
+ */
198
+ export async function playMidi(
199
+ midiBytes: ArrayBuffer,
200
+ ): Promise<{ stop: () => void; done: Promise<void> }> {
201
+ const ctx = new AudioContext();
202
+ await ctx.resume();
203
+
204
+ const master = ctx.createGain();
205
+ master.gain.value = 0.8;
206
+ master.connect(ctx.destination);
207
+
208
+ const notes = parseMidi(midiBytes);
209
+ const startAt = ctx.currentTime + 0.1;
210
+ for (const note of notes) {
211
+ scheduleNote(ctx, note, master, startAt + note.time);
212
+ }
213
+ const lastMidiTime = notes.length ? (notes[notes.length - 1]?.time ?? 0) : 0;
214
+ const end = startAt + lastMidiTime + 1;
215
+
216
+ const done = new Promise<void>((resolve) => {
217
+ setTimeout(() => {
218
+ try {
219
+ ctx.close();
220
+ } catch {
221
+ /* already closed */
222
+ }
223
+ resolve();
224
+ }, Math.max(0, (end - ctx.currentTime) * 1000) + 200);
225
+ });
226
+
227
+ return {
228
+ stop: () => {
229
+ try {
230
+ ctx.close();
231
+ } catch {
232
+ /* ignore */
233
+ }
234
+ },
235
+ done,
236
+ };
237
+ }
238
+
239
  /**
240
  * Get the total duration (seconds) of a MIDI file β€” the time of the last note
241
  * end. This is what the generated response must match so the take and the
apps/web/lib/jambuddy/visualizer.tsx CHANGED
@@ -12,9 +12,14 @@
12
  * you can compare the take and the response side by side.
13
  */
14
 
15
- import { useEffect, useRef } from "react";
16
  import { parseMidi, type ParsedNote } from "./player";
17
 
 
 
 
 
 
18
  interface VisualizerProps {
19
  /** MIDI bytes -> piano-roll. Mutually exclusive with audioUrl. */
20
  midiBytes?: ArrayBuffer | null;
@@ -24,6 +29,11 @@ interface VisualizerProps {
24
  label?: string;
25
  /** Height of the canvas in px. */
26
  height?: number;
 
 
 
 
 
27
  }
28
 
29
  const NOTE_MIN = 21; // A0
@@ -119,77 +129,143 @@ function drawWaveform(
119
  }
120
  }
121
 
122
- export function Visualizer({
123
- midiBytes,
124
- audioUrl,
125
- label,
126
- height = 120,
127
- }: VisualizerProps) {
128
- const canvasRef = useRef<HTMLCanvasElement>(null);
129
-
130
- useEffect(() => {
131
- const canvas = canvasRef.current;
132
- if (!canvas) return;
133
-
134
- if (midiBytes) {
135
- let notes: ParsedNote[] = [];
136
- let duration = 0;
137
- try {
138
- notes = parseMidi(midiBytes);
139
- duration = notes.reduce(
140
- (m, n) => Math.max(m, n.time + n.duration),
141
- 0,
142
- );
143
- } catch {
144
- /* corrupt bytes -> empty roll */
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
145
  }
146
- drawPianoRoll(canvas, notes, duration);
147
- return;
148
- }
149
 
150
- if (audioUrl) {
151
- const ctx = new AudioContext();
152
- fetch(audioUrl)
153
- .then((r) => r.arrayBuffer())
154
- .then((buf) => ctx.decodeAudioData(buf))
155
- .then((audio) => {
156
- drawWaveform(canvas, audio);
157
- ctx.close();
158
- })
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
159
  .catch(() => {
160
- const c = canvas.getContext("2d");
161
- if (c) {
162
- c.fillStyle = "#12131b";
163
- c.fillRect(0, 0, canvas.clientWidth, canvas.clientHeight);
164
- c.fillStyle = "#7f829c";
165
- c.font = "12px ui-monospace, monospace";
166
- c.textAlign = "center";
167
- c.fillText("no audio", canvas.clientWidth / 2, canvas.clientHeight / 2);
168
- }
169
  });
170
- return;
171
- }
172
 
173
- // Nothing to draw.
174
- const c = canvas.getContext("2d");
175
- if (c) {
176
- c.fillStyle = "#12131b";
177
- c.fillRect(0, 0, canvas.clientWidth, canvas.clientHeight);
178
- }
179
- }, [midiBytes, audioUrl]);
180
-
181
- return (
182
- <div className="w-full">
183
- {label && (
184
- <div className="mb-1 font-mono text-[10px] uppercase tracking-widest text-[#7f829c]">
185
- {label}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
186
  </div>
187
- )}
188
- <canvas
189
- ref={canvasRef}
190
- className="w-full rounded border border-[#2a2d3d] bg-[#12131b]"
191
- style={{ height }}
192
- />
193
- </div>
194
- );
195
- }
 
12
  * you can compare the take and the response side by side.
13
  */
14
 
15
+ import { forwardRef, useEffect, useImperativeHandle, useRef, useState } from "react";
16
  import { parseMidi, type ParsedNote } from "./player";
17
 
18
+ export interface VisualizerHandle {
19
+ /** Stop any audio this visualizer is playing (used to enforce exclusivity). */
20
+ stop: () => void;
21
+ }
22
+
23
  interface VisualizerProps {
24
  /** MIDI bytes -> piano-roll. Mutually exclusive with audioUrl. */
25
  midiBytes?: ArrayBuffer | null;
 
29
  label?: string;
30
  /** Height of the canvas in px. */
31
  height?: number;
32
+ /** Show a play/stop toggle on the waveform (audio only). */
33
+ playable?: boolean;
34
+ /** Called right before this visualizer starts playing, so the owner can stop
35
+ * any other audio source (enforces "one waveform, one playback"). */
36
+ onStartPlayback?: () => void;
37
  }
38
 
39
  const NOTE_MIN = 21; // A0
 
129
  }
130
  }
131
 
132
+ export const Visualizer = forwardRef<VisualizerHandle, VisualizerProps>(
133
+ function Visualizer(
134
+ {
135
+ midiBytes,
136
+ audioUrl,
137
+ label,
138
+ height = 120,
139
+ playable = false,
140
+ onStartPlayback,
141
+ },
142
+ ref,
143
+ ) {
144
+ const canvasRef = useRef<HTMLCanvasElement>(null);
145
+ // Play/stop state for the waveform toggle (audio only).
146
+ const [playing, setPlaying] = useState(false);
147
+ const audioRef = useRef<HTMLAudioElement | null>(null);
148
+
149
+ // Expose a stop() so the parent can enforce one-waveform-at-a-time.
150
+ useImperativeHandle(ref, () => ({
151
+ stop: () => {
152
+ audioRef.current?.pause();
153
+ audioRef.current = null;
154
+ setPlaying(false);
155
+ },
156
+ }));
157
+
158
+ useEffect(() => {
159
+ const canvas = canvasRef.current;
160
+ if (!canvas) return;
161
+ // reset play state whenever the audio changes
162
+ setPlaying(false);
163
+ if (audioRef.current) {
164
+ audioRef.current.pause();
165
+ audioRef.current = null;
166
+ }
167
+ if (midiBytes) {
168
+ let notes: ParsedNote[] = [];
169
+ let duration = 0;
170
+ try {
171
+ notes = parseMidi(midiBytes);
172
+ duration = notes.reduce(
173
+ (m, n) => Math.max(m, n.time + n.duration),
174
+ 0,
175
+ );
176
+ } catch {
177
+ /* corrupt bytes -> empty roll */
178
+ }
179
+ drawPianoRoll(canvas, notes, duration);
180
+ return;
181
+ }
182
+
183
+ if (audioUrl) {
184
+ const ctx = new AudioContext();
185
+ fetch(audioUrl)
186
+ .then((r) => r.arrayBuffer())
187
+ .then((buf) => ctx.decodeAudioData(buf))
188
+ .then((audio) => {
189
+ drawWaveform(canvas, audio);
190
+ ctx.close();
191
+ })
192
+ .catch(() => {
193
+ const c = canvas.getContext("2d");
194
+ if (c) {
195
+ c.fillStyle = "#12131b";
196
+ c.fillRect(0, 0, canvas.clientWidth, canvas.clientHeight);
197
+ c.fillStyle = "#7f829c";
198
+ c.font = "12px ui-monospace, monospace";
199
+ c.textAlign = "center";
200
+ c.fillText("no audio", canvas.clientWidth / 2, canvas.clientHeight / 2);
201
+ }
202
+ });
203
+ return;
204
  }
 
 
 
205
 
206
+ // Nothing to draw.
207
+ const c = canvas.getContext("2d");
208
+ if (c) {
209
+ c.fillStyle = "#12131b";
210
+ c.fillRect(0, 0, canvas.clientWidth, canvas.clientHeight);
211
+ }
212
+ }, [midiBytes, audioUrl]);
213
+
214
+ // Toggle play/stop of the waveform's audio (audio only).
215
+ const togglePlay = () => {
216
+ if (!audioUrl) return;
217
+ if (playing) {
218
+ audioRef.current?.pause();
219
+ audioRef.current = null;
220
+ setPlaying(false);
221
+ return;
222
+ }
223
+ // Tell the owner to stop anything else first (one playback at a time).
224
+ onStartPlayback?.();
225
+ const a = new Audio(audioUrl);
226
+ a.onended = () => {
227
+ audioRef.current = null;
228
+ setPlaying(false);
229
+ };
230
+ audioRef.current = a;
231
+ a.play()
232
+ .then(() => setPlaying(true))
233
  .catch(() => {
234
+ audioRef.current = null;
235
+ setPlaying(false);
 
 
 
 
 
 
 
236
  });
237
+ };
 
238
 
239
+ return (
240
+ <div className="w-full">
241
+ {label && (
242
+ <div className="mb-1 font-mono text-[10px] uppercase tracking-widest text-[#7f829c]">
243
+ {label}
244
+ </div>
245
+ )}
246
+ <div className="flex items-center gap-2">
247
+ {playable && audioUrl && (
248
+ <button
249
+ type="button"
250
+ onClick={togglePlay}
251
+ aria-label={playing ? "Stop playback" : "Play"}
252
+ title={playing ? "Stop" : "Play"}
253
+ className={`grid h-9 w-9 shrink-0 place-items-center rounded-full border text-sm ${
254
+ playing
255
+ ? "border-[#e05252] bg-[#e05252] text-[#12131b]"
256
+ : "border-[#2a2d3d] bg-[#1a1c28] text-[#5fd38a] hover:bg-[#5fd38a] hover:text-[#12131b]"
257
+ }`}
258
+ >
259
+ {playing ? "β– " : "β–Ά"}
260
+ </button>
261
+ )}
262
+ <canvas
263
+ ref={canvasRef}
264
+ className="w-full rounded border border-[#2a2d3d] bg-[#12131b]"
265
+ style={{ height }}
266
+ />
267
  </div>
268
+ </div>
269
+ );
270
+ },
271
+ );
 
 
 
 
 
docs/09-risks.md CHANGED
@@ -312,3 +312,54 @@ Risks ranked by impact Γ— likelihood. Each has an owner, a mitigation, and a con
312
  - Mobile apps
313
  - Cloud accounts / user authentication
314
  - Anything that requires a third-party API key beyond SA3
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
312
  - Mobile apps
313
  - Cloud accounts / user authentication
314
  - Anything that requires a third-party API key beyond SA3
315
+
316
+ ---
317
+
318
+ ## Current Jam Buddy bugs & findings (verified against Stability API docs)
319
+
320
+ > These are open/confirmed items tracked for the current work. Docs consulted:
321
+ > `platform.stability.ai/docs/api-reference` β†’ `post /v2beta/audio/stable-audio/audio-to-audio`.
322
+
323
+ ### 1. SA3 `strength` (noise/denoising) is NOT being sent β€” open bug (high priority)
324
+
325
+ The SA3 **audio-to-audio** request schema exposes a `strength` parameter (a.k.a.
326
+ *denoising*): **0 = output identical to input, 1 = as if no input was given.**
327
+ We do **not** send it, so it defaults to **1** = full diffusion. This explains:
328
+ - "generations render nothing new" (input take fully morphed / barely anchors output)
329
+ - "output contains more than one instrument" (the take's instrument doesn't anchor,
330
+ model freely adds others)
331
+
332
+ Stability's guidance for audio-to-audio is `strength` β‰ˆ **0.5–0.8** (diffuse the
333
+ take but keep it as the anchor). This is the pending **"Noise knob"** task β€”
334
+ wire `strength` into `tools/jam_buddy_api.py` and expose it in the UI. **UNSENT today.**
335
+
336
+ ### 2. The audio API has NO `negative_prompt` β€” confirmed
337
+ SA3 audio-to-audio only accepts: `prompt`, `audio`, `model`, `duration`, `seed`,
338
+ `steps`, `cfg_scale`, `output_format`, `strength`. There is **no `negative_prompt`**.
339
+ (Negative prompts exist on Stable *Image*, not Stable Audio.) So we **cannot** steer
340
+ away from extra instruments via a negative prompt β€” the positive prompt + `cfg_scale`
341
+ + `strength` are the only levers. The local CPU fallback (`small-music`) DOES accept
342
+ a negative prompt.
343
+
344
+ ### 3. Demo examples can't be played β€” RESOLVED
345
+ The clickable demo chips (Gradio-style) load MIDI as a take, but preview playback
346
+ doesn't work β€” the MIDI examples need to render to audio to actually be audible
347
+ as a "playable example." Consider pre-rendering the demo MIDIs to audio (or
348
+ ensuring the Web-MIDI synth preview actually plays).
349
+
350
+ **Resolved:** demos now load their pre-rendered MP3 as an audio take (with the
351
+ BPM knob pinned to the source MIDI's tempo), and playback is a play/stop toggle
352
+ on the waveform. Single-select + instant highlight/knob update.
353
+
354
+ ### 4. Stale `.next` cache corrupts the dev server (recurring) β€” OPEN
355
+ `next dev`'s incremental cache corrupts after heavy edits. Symptoms: API routes
356
+ 500 (`MODULE_NOT_FOUND` in `webpack-runtime.js`) OR the page renders blank
357
+ (every `/_next/static/chunk` 404s while `GET /` still returns 200). Often a
358
+ leftover process squats on port 3000. Fix today: kill the port-3000 PID
359
+ (`netstat -ano | grep :3000`, `taskkill /F /PID`), `rm -rf apps/web/.next`,
360
+ restart `pnpm dev`, verify chunks load (not just `GET /`). **Address later** β€”
361
+ candidate: a `dev:clean` npm script that clears `.next` before starting, or a
362
+ more robust dev workflow.
363
+
364
+
365
+
tools/midi_to_audio.py CHANGED
@@ -12,7 +12,7 @@ guitar reference.
12
 
13
  Usage:
14
  .venv/Scripts/python.exe tools/midi_to_audio.py \
15
- --midi tools/hextermination_blast.mid \
16
  --oneshots tools/oneshots \
17
  --bpm 184 --out drums.wav
18
  # add --guitar ref.wav to mix drums under the guitar
 
12
 
13
  Usage:
14
  .venv/Scripts/python.exe tools/midi_to_audio.py \
15
+ --midi demos/tupatutupatututata.mid \
16
  --oneshots tools/oneshots \
17
  --bpm 184 --out drums.wav
18
  # add --guitar ref.wav to mix drums under the guitar