kale-tts-1 / visual.js
NERDDISCO's picture
kale-tts-1
624a22c
Raw History Blame Contribute Delete
13.7 kB
// Visuals for the kale-tts-1-nano space: an audio-reactive orb (WebGL2 ray-marched sphere) and glowing voice lines
// (64 log-spaced analyser bands, eased fast up / slower down, blended 1-2-1, laid along the line
// under a sine window; and an oscilloscope line from the time-domain samples, started at a rising zero crossing).
// Nothing animates unless audio plays: one requestAnimationFrame loop runs while sound is on and stops ~0.8 s after it ends.
const REDUCED = matchMedia("(prefers-reduced-motion: reduce)");
// ------------------------------------------------------------------------------------------ analyser -> bands / scope
export const BANDS = 64;
function edges(n, top, start = 3) {
const e = []; for (let j = 0; j <= n; j++) e.push(Math.max(j, Math.round(start * Math.pow(top / start, j / n))));
for (let j = 1; j <= n; j++) e[j] = Math.max(e[j], e[j - 1] + 1); return e;
}
export class Listener {
constructor(analyser) {
this.a = analyser; this.freq = new Uint8Array(analyser.frequencyBinCount); this.time = new Float32Array(analyser.fftSize);
// voice range only: up to ~5 kHz of the 12 kHz Nyquist at 24 kHz
this.edges = edges(BANDS, Math.round(analyser.frequencyBinCount * (5000 / (analyser.context.sampleRate / 2))));
this.bands = new Float32Array(BANDS); this.soft = new Float32Array(BANDS); this.level = 0; this.rms = 0;
}
update(dt) {
this.a.getByteFrequencyData(this.freq); this.a.getFloatTimeDomainData(this.time);
const n = BANDS, floor = 0.34, tilt = 0.7;
for (let j = 0; j < n; j++) {
let v = 0; for (let k = this.edges[j]; k < this.edges[j + 1]; k++) if (this.freq[k] > v) v = this.freq[k];
v = Math.max(0, (v / 255 - floor) / (1 - floor)) * (1 + tilt * (j / (n - 1))); v = 1 - Math.exp(-2.4 * v * v);
this.bands[j] += (v - this.bands[j]) * (1 - Math.exp(-(v > this.bands[j] ? 46 : 16) * dt));
}
for (let j = 0; j < n; j++) this.soft[j] = (this.bands[Math.max(0, j - 1)] + 2 * this.bands[j] + this.bands[Math.min(n - 1, j + 1)]) / 4;
let sq = 0; for (let i = 0; i < this.time.length; i++) sq += this.time[i] * this.time[i]; this.rms = Math.sqrt(sq / this.time.length);
const target = Math.min(1, this.rms * 7); this.level += (target - this.level) * (1 - Math.exp(-(target > this.level ? 30 : 6) * dt));
}
/** the scope line's points (-1..1) into out: a ~24 ms window from a rising zero crossing, gain + soft clip, gated by RMS */
scope(out, rate, gain = 4) {
const s = this.time, n = s.length, m = out.length, win = Math.max(m, Math.min(n, Math.round(rate * 0.024)));
let trig = 0, low = false; const hys = this.rms * 0.3;
for (let i = 1; i < n - win; i++) { if (s[i] < -hys) low = true; else if (low && s[i - 1] < 0 && s[i] >= 0) { trig = i; break; } }
const g = Math.min(1, Math.max(0, (this.rms - 0.0015) / 0.0045)), gate = g * g * (3 - 2 * g), step = win / m;
for (let p = 0; p < m; p++) { const a = trig + Math.floor(p * step), b = Math.max(a + 1, trig + Math.floor((p + 1) * step)); let v = 0;
for (let i = a; i < b; i++) v += s[i]; v /= b - a; out[p] = (v < 0 ? -1 : 1) * Math.tanh(Math.pow(gain * Math.abs(v), 0.7)) * gate; }
}
}
const cr = (a, p) => { const n = a.length, i = Math.floor(p), t = p - i, at = (k) => a[k < 0 ? 0 : k >= n ? n - 1 : k];
const p0 = at(i - 1), p1 = at(i), p2 = at(i + 1), p3 = at(i + 2); return p1 + 0.5 * t * (p2 - p0 + t * (2 * p0 - 5 * p1 + 4 * p2 - p3 + t * (3 * (p1 - p2) + p3 - p0))); };
const win = (u) => Math.pow(Math.sin(Math.PI * u), 0.6);
// ------------------------------------------------------------------------------------------ 2D lines (hero line + cards)
function fit(cv) { const dpr = Math.min(2, devicePixelRatio || 1), w = cv.clientWidth, h = cv.clientHeight;
if (cv.width !== Math.round(w * dpr) || cv.height !== Math.round(h * dpr)) { cv.width = Math.round(w * dpr); cv.height = Math.round(h * dpr); }
const g = cv.getContext("2d"); g.setTransform(dpr, 0, 0, dpr, 0, 0); g.clearRect(0, 0, w, h); return { g, w, h }; }
function glowStroke(g, path, color, width, blur, alpha = 1) {
g.save(); g.globalAlpha = alpha; g.strokeStyle = color; g.lineWidth = width; g.lineJoin = g.lineCap = "round"; g.shadowColor = color; g.shadowBlur = blur; g.stroke(path); g.restore(); }
/** the hero's voice line: the live scope along the width, the band shape as a faint ghost; flat and quiet when nothing plays */
export function drawHeroLine(cv, L, color, active) {
const { g, w, h } = fit(cv), mid = h / 2, pts = 220, sc = drawHeroLine.sc ??= new Float32Array(pts);
if (active && L) L.scope(sc, 24000, 5); else sc.fill(0);
const ghost = new Path2D(), line = new Path2D();
for (let i = 0; i <= pts; i++) { const u = i / pts, x = u * w, b = L && active ? cr(L.soft, u * BANDS - 0.5) : 0;
const yG = mid - b * win(u) * h * 0.42; i ? ghost.lineTo(x, yG) : ghost.moveTo(x, yG);
const y = mid - sc[Math.min(pts - 1, i)] * 0.8 * win(u) * h * 0.46; i ? line.lineTo(x, y) : line.moveTo(x, y); }
for (let i = pts; i >= 0; i--) { const u = i / pts, b = L && active ? cr(L.soft, u * BANDS - 0.5) : 0; ghost.lineTo(u * w, mid + b * win(u) * h * 0.42); }
ghost.closePath();
const grad = g.createLinearGradient(0, 0, w, 0); grad.addColorStop(0, "transparent"); grad.addColorStop(0.5, color); grad.addColorStop(1, "transparent");
g.save(); g.globalAlpha = active ? 0.16 : 0.06; g.fillStyle = grad; g.fill(ghost); g.restore();
glowStroke(g, line, color, 2.2, 18, active ? 1 : 0.35); glowStroke(g, line, "#ffffff", 0.9, 0, active ? 0.75 : 0.2);
}
/** a voice card's waveform: the real samples of that voice's rendered audio, peak (light) and RMS (solid) per 3 px column,
* mirrored around the middle. progress null = idle (whole wave in the voice colour); 0..1 = playing: played part tinted,
* the rest grey, one playhead line. Nothing else moves. */
const _cols = new WeakMap();
function columns(audio, n) {
const hit = _cols.get(audio); if (hit && hit.n === n) return hit;
const peak = new Float32Array(n), rms = new Float32Array(n), step = audio.length / n; let top = 1e-4;
for (let i = 0; i < n; i++) { let m = 0, q = 0; const a = Math.floor(i * step), b = Math.max(a + 1, Math.floor((i + 1) * step));
for (let j = a; j < b; j++) { const v = Math.abs(audio[j]); if (v > m) m = v; q += v * v; } peak[i] = m; rms[i] = Math.sqrt(q / (b - a)); if (m > top) top = m; }
for (let i = 0; i < n; i++) { peak[i] /= top; rms[i] /= top; }
const out = { n, peak, rms }; _cols.set(audio, out); return out;
}
export function drawCardLine(cv, audio, color, progress = null) {
const { g, w, h } = fit(cv), mid = h / 2;
if (!audio) { g.fillStyle = "rgba(255,255,255,.16)"; g.fillRect(0, mid - 0.5, w, 1); return; }
const n = Math.max(8, Math.floor(w / 3)), { peak, rms } = columns(audio, n), bw = w / n, px = progress === null ? w : Math.max(0, Math.min(1, progress)) * w;
for (let i = 0; i < n; i++) {
const x = i * bw, on = x + bw / 2 <= px, hp = Math.max(1, peak[i] * (h - 2) * 0.5), hr = Math.min(hp, Math.max(1, rms[i] * (h - 2) * 0.8));
g.fillStyle = on ? color : "rgba(255,255,255,.22)"; g.globalAlpha = on ? 0.38 : 0.5; g.fillRect(x, mid - hp, Math.max(1, bw - 1), hp * 2);
g.globalAlpha = on ? (progress === null ? 0.8 : 1) : 0.75; g.fillRect(x, mid - hr, Math.max(1, bw - 1), hr * 2);
}
g.globalAlpha = 1;
if (progress !== null && progress > 0 && progress < 1) { g.fillStyle = "#fff"; g.fillRect(Math.round(px) - 0.75, 1, 1.5, h - 2); }
}
// ------------------------------------------------------------------------------------------ the orb (WebGL2)
const VS = `#version 300 es
void main(){ vec2 p = vec2((gl_VertexID << 1) & 2, gl_VertexID & 2); gl_Position = vec4(p * 2.0 - 1.0, 0.0, 1.0); }`;
const FS = `#version 300 es
precision highp float;
uniform vec2 uRes; uniform float uTime, uLevel, uSpeak; uniform vec3 uCol, uCol2; uniform float uB[16];
out vec4 o;
float band(float y){ float f = clamp(y * 0.5 + 0.5, 0.0, 1.0) * 15.0; int i = int(floor(f)); return mix(uB[i], uB[min(i + 1, 15)], fract(f)); }
mat2 rot(float a){ float c = cos(a), s = sin(a); return mat2(c, -s, s, c); }
float wav(vec3 p, float t){ return sin(p.x * 3.1 + t) * sin(p.y * 2.7 - t * 0.8) * sin(p.z * 3.3 + t * 0.6)
+ 0.5 * sin(p.x * 6.3 - t * 1.3 + p.z * 2.0) * sin(p.y * 5.9 + t * 1.1); }
float map(vec3 p){
vec3 q = p; q.xz *= rot(uTime * 0.25); q.xy *= rot(0.35);
float b = band(normalize(q).y);
float r = 0.74 + 0.06 * uLevel;
float d = wav(q * (1.15 + 0.25 * uSpeak), uTime * 1.2) * (0.016 + 0.05 * b * uSpeak + 0.022 * uLevel);
return length(p) - r - d;
}
vec3 nrm(vec3 p){ vec2 e = vec2(0.0025, 0.0); return normalize(vec3(map(p + e.xyy) - map(p - e.xyy), map(p + e.yxy) - map(p - e.yxy), map(p + e.yyx) - map(p - e.yyx))); }
float hash(vec2 p){ return fract(sin(dot(p, vec2(12.9898, 78.233))) * 43758.5453); }
void main(){
vec2 uv = (gl_FragCoord.xy - 0.5 * uRes) / min(uRes.x, uRes.y);
vec3 ro = vec3(0.0, 0.0, 3.2), rd = normalize(vec3(uv * 1.05, -1.6));
float r = 0.74 + 0.06 * uLevel;
// halo from the ray's closest approach to the centre: smooth, no per-step sums (those drew the rings / layers)
float tc = max(dot(-ro, rd), 0.0), bd = length(ro + rd * tc);
float g = exp(-max(bd - r, 0.0) * (6.5 - 2.0 * uLevel)) * (0.32 + 0.5 * uLevel + 0.12 * uSpeak);
vec3 halo = mix(uCol, uCol2, 0.35) * g;
float dith = (hash(gl_FragCoord.xy + fract(uTime)) - 0.5) / 255.0;
// march only rays that can reach the shape's shell; relative hit threshold; then refine onto the surface
float t = 0.0, hit = 0.0;
if (bd < r + 0.2) {
t = max(0.0, tc - (r + 0.25));
for (int i = 0; i < 64; i++) { float d = map(ro + rd * t); if (abs(d) < 0.0004 * t) { hit = 1.0; break; } t += d * 0.6; if (t > 6.0) break; }
if (hit > 0.5) for (int k = 0; k < 3; k++) t += map(ro + rd * t);
}
vec2 ed = min(gl_FragCoord.xy, uRes - gl_FragCoord.xy) / min(uRes.x, uRes.y); float edge = smoothstep(0.0, 0.18, min(ed.x, ed.y)); g *= edge; halo *= edge;
if (hit < 0.5) { o = vec4(halo + dith, clamp(g, 0.0, 1.0)); return; }
vec3 p = ro + rd * t, n = nrm(p), l = normalize(vec3(-0.5, 0.7, 0.6));
float fr = pow(1.0 - max(dot(n, -rd), 0.0), 2.4), dif = max(dot(n, l), 0.0), spec = pow(max(dot(reflect(-l, n), -rd), 0.0), 40.0);
vec3 irid = 0.5 + 0.5 * cos(6.2831 * (vec3(0.0, 0.33, 0.67) + n.y * 0.35 + fr * 0.6 + uTime * 0.05));
vec3 base = mix(vec3(0.02, 0.03, 0.05), uCol * 0.35, 0.35 + 0.65 * dif);
vec3 col = base + fr * mix(uCol, irid, 0.35) * (1.1 + 1.2 * uLevel) + spec * vec3(1.0) * 0.8 + uCol2 * 0.12 * (1.0 - fr) + halo * 0.25;
o = vec4(col + dith, 1.0);
}`;
export class Orb {
constructor(canvas) {
this.cv = canvas; const gl = canvas.getContext("webgl2", { premultipliedAlpha: true, alpha: true, antialias: false, powerPreference: "high-performance" });
this.ok = !!gl; if (!gl) return; this.gl = gl;
const sh = (t, s) => { const x = gl.createShader(t); gl.shaderSource(x, s); gl.compileShader(x); if (!gl.getShaderParameter(x, gl.COMPILE_STATUS)) throw new Error(gl.getShaderInfoLog(x)); return x; };
const p = gl.createProgram(); gl.attachShader(p, sh(gl.VERTEX_SHADER, VS)); gl.attachShader(p, sh(gl.FRAGMENT_SHADER, FS)); gl.linkProgram(p); gl.useProgram(p); this.p = p;
this.u = Object.fromEntries(["uRes", "uTime", "uLevel", "uSpeak", "uCol", "uCol2", "uB"].map((k) => [k, gl.getUniformLocation(p, k)]));
this.t = 0; this.col = [0.2, 0.83, 0.6]; this.col2 = [0.15, 0.4, 0.9]; this.b16 = new Float32Array(16); this.speak = 0;
this.scale = matchMedia("(max-width: 760px)").matches ? 0.75 : 1; // full resolution on desktop: no upscaled blur
}
setColor(hex, hex2 = "#3b82f6") { const c = (h) => [1, 3, 5].map((i) => parseInt(h.slice(i, i + 2), 16) / 255); this.col = c(hex); this.col2 = c(hex2); }
draw(L, dt, active) {
if (!this.ok) return; const gl = this.gl, dpr = Math.min(1.5, devicePixelRatio || 1) * this.scale;
const w = Math.round(this.cv.clientWidth * dpr), h = Math.round(this.cv.clientHeight * dpr); if (this.cv.width !== w || this.cv.height !== h) { this.cv.width = w; this.cv.height = h; }
const moving = active && !REDUCED.matches; if (moving) this.t += dt * (0.6 + 1.4 * (L?.level ?? 0));
this.speak += ((active ? 1 : 0) - this.speak) * Math.min(1, dt * 4);
if (L) for (let i = 0; i < 16; i++) { let s = 0; for (let k = 0; k < 4; k++) s += L.soft[i * 4 + k]; this.b16[i] = REDUCED.matches ? 0 : s / 4; }
gl.viewport(0, 0, w, h); gl.clearColor(0, 0, 0, 0); gl.clear(gl.COLOR_BUFFER_BIT);
gl.uniform2f(this.u.uRes, w, h); gl.uniform1f(this.u.uTime, this.t); gl.uniform1f(this.u.uLevel, L ? L.level : 0); gl.uniform1f(this.u.uSpeak, REDUCED.matches ? 0 : this.speak);
gl.uniform3fv(this.u.uCol, this.col); gl.uniform3fv(this.u.uCol2, this.col2); gl.uniform1fv(this.u.uB, this.b16); gl.drawArrays(gl.TRIANGLES, 0, 3);
}
}
// ------------------------------------------------------------------------------------------ the on-demand loop
/** frame(dt, active) is called every animation frame while `isPlaying()` is true and for `tail` seconds after; then once more, idle. */
export function onDemandLoop(frame, isPlaying, tail = 0.8) {
let raf = 0, last = 0, quiet = 0;
const step = (now) => { const dt = Math.min(0.05, (now - last) / 1000 || 0.016); last = now; const on = isPlaying();
quiet = on ? 0 : quiet + dt; frame(dt, on || quiet < tail);
if (quiet < tail) raf = requestAnimationFrame(step); else { raf = 0; frame(0, false); } };
return { kick() { if (!raf) { last = performance.now(); quiet = 0; raf = requestAnimationFrame(step); } }, redraw() { if (!raf) frame(0, false); }, get running() { return !!raf; } };
}
export const reducedMotion = REDUCED;