feat: fast MP4 video generation via WebCodecs + mp4-muxer

index.html

@@ -5,7 +5,8 @@
   <meta name="viewport" content="width=device-width, initial-scale=1.0" />
   <title>ace-step-jam</title>
   <link rel="preconnect" href="https://fonts.googleapis.com" />
-  <link href="https://fonts.googleapis.com/css2?family=Hanken+Grotesk:wght@300;400;500;600&display=swap" rel="stylesheet" />
+  <link href="https://fonts.googleapis.com/css2?family=Hanken+Grotesk:wght@300;400;500;600;700&display=swap" rel="stylesheet" />
+  <script src="https://cdn.jsdelivr.net/npm/mp4-muxer@5.1.3/build/mp4-muxer.min.js"></script>
   <style>
     *, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
 
@@ -437,7 +438,9 @@
     // ── Current song state (for video generation) ────────────────────────────
     let currentSong = null;
 
-    // ── Video generation ────────────────────────────────────────────────────
+    // ── Fast video generation (WebCodecs + MP4 muxer) ─────────────────────
+    const { Muxer: Mp4Muxer, ArrayBufferTarget } = Mp4Muxer;
+
     async function generateShareVideo(song, btn) {
       const origHTML = btn.innerHTML;
       btn.disabled = true;
@@ -461,8 +464,12 @@
 
         const audioResp = await fetch(song.audio_url);
         const audioArrayBuf = await audioResp.arrayBuffer();
-        const audioCtx = new AudioContext();
-        const decoded = await audioCtx.decodeAudioData(audioArrayBuf);
+        const offCtx = new OfflineAudioContext(2, 1, 44100);
+        const decoded = await offCtx.decodeAudioData(audioArrayBuf);
+
+        const sampleRate = decoded.sampleRate;
+        const numChannels = decoded.numberOfChannels;
+        const audioLength = decoded.length;
 
         const raw = decoded.getChannelData(0);
         const numBars = 60;
@@ -486,28 +493,11 @@
         const W = 1080, H = 1080;
         const canvas = document.createElement("canvas");
         canvas.width = W; canvas.height = H;
-        const ctx = canvas.getContext("2d");
-
-        const source = audioCtx.createBufferSource();
-        source.buffer = decoded;
-        const audioDest = audioCtx.createMediaStreamDestination();
-        source.connect(audioDest);
-
-        const canvasStream = canvas.captureStream(30);
-        const combined = new MediaStream([
-          ...canvasStream.getTracks(),
-          ...audioDest.stream.getTracks(),
-        ]);
-
-        const recorder = new MediaRecorder(combined, {
-          mimeType: MediaRecorder.isTypeSupported("video/webm;codecs=vp9,opus")
-            ? "video/webm;codecs=vp9,opus" : "video/webm",
-          videoBitsPerSecond: 2_000_000,
-        });
-        const chunks = [];
-        recorder.ondataavailable = e => { if (e.data.size > 0) chunks.push(e.data); };
+        const ctx = canvas.getContext("2d", { willReadFrequently: true });
 
         const duration = decoded.duration;
+        const FPS = 15;
+        const totalFrames = Math.ceil(duration * FPS);
 
         function drawFrame(progress) {
           ctx.fillStyle = "#111111";
@@ -563,33 +553,78 @@
           ctx.textAlign = "left";
         }
 
-        recorder.start(100);
-        source.start();
-        const startT = performance.now();
+        // ── Encode with WebCodecs + MP4 muxer ──
+        const target = new ArrayBufferTarget();
+        const muxer = new Mp4Muxer({
+          target,
+          video: { codec: "avc", width: W, height: H },
+          audio: { codec: "aac", sampleRate, numberOfChannels: numChannels },
+          firstTimestampBehavior: "offset",
+          fastStart: "in-memory",
+        });
+
+        const audioEncoder = new AudioEncoder({
+          output: (chunk, meta) => muxer.addAudioChunk(chunk, meta),
+          error: e => console.error("AudioEncoder error:", e),
+        });
+        audioEncoder.configure({ codec: "mp4a.40.2", sampleRate, numberOfChannels: numChannels, bitrate: 128_000 });
+
+        const audioChunkSize = 4096;
+        for (let offset = 0; offset < audioLength; offset += audioChunkSize) {
+          const len = Math.min(audioChunkSize, audioLength - offset);
+          const audioData = new AudioData({
+            format: "f32-planar", sampleRate, numberOfFrames: len, numberOfChannels: numChannels,
+            timestamp: (offset / sampleRate) * 1_000_000,
+            data: (() => {
+              const buf = new Float32Array(len * numChannels);
+              for (let ch = 0; ch < numChannels; ch++) {
+                buf.set(decoded.getChannelData(ch).subarray(offset, offset + len), ch * len);
+              }
+              return buf;
+            })(),
+          });
+          audioEncoder.encode(audioData);
+          audioData.close();
+        }
+        await audioEncoder.flush();
+
+        const videoEncoder = new VideoEncoder({
+          output: (chunk, meta) => muxer.addVideoChunk(chunk, meta),
+          error: e => console.error("VideoEncoder error:", e),
+        });
+        videoEncoder.configure({ codec: "avc1.640028", width: W, height: H, bitrate: 2_000_000, framerate: FPS });
 
+        let frameIdx = 0;
+        const batchSize = 5;
         await new Promise((resolve) => {
-          function tick() {
-            const elapsed = (performance.now() - startT) / 1000;
-            const progress = Math.min(elapsed / duration, 1);
-            drawFrame(progress);
-            pctEl.textContent = `${Math.round(progress * 100)}%`;
-            barEl.style.width = `${Math.round(progress * 100)}%`;
-            if (progress < 1) requestAnimationFrame(tick);
-            else setTimeout(() => recorder.stop(), 200);
+          function processBatch() {
+            const end = Math.min(frameIdx + batchSize, totalFrames);
+            for (; frameIdx < end; frameIdx++) {
+              const progress = frameIdx / totalFrames;
+              drawFrame(progress);
+              const frame = new VideoFrame(canvas, { timestamp: (frameIdx / FPS) * 1_000_000 });
+              videoEncoder.encode(frame, { keyFrame: frameIdx % (FPS * 2) === 0 });
+              frame.close();
+            }
+            pctEl.textContent = `${Math.round((frameIdx / totalFrames) * 100)}%`;
+            barEl.style.width = `${Math.round((frameIdx / totalFrames) * 100)}%`;
+            if (frameIdx < totalFrames) setTimeout(processBatch, 0);
+            else {
+              videoEncoder.flush().then(() => {
+                muxer.finalize();
+                const blob = new Blob([target.buffer], { type: "video/mp4" });
+                const url = URL.createObjectURL(blob);
+                const a = document.createElement("a");
+                a.href = url;
+                const safeName = (song.title || "song").replace(/[^a-zA-Z0-9 -]/g, "").trim().replace(/\s+/g, "-") || "song";
+                a.download = `${safeName}.mp4`;
+                a.click();
+                URL.revokeObjectURL(url);
+                resolve();
+              });
+            }
           }
-          recorder.onstop = () => {
-            const blob = new Blob(chunks, { type: "video/webm" });
-            const url = URL.createObjectURL(blob);
-            const a = document.createElement("a");
-            a.href = url;
-            const safeName = (song.title || "song").replace(/[^a-zA-Z0-9 -]/g, "").trim().replace(/\s+/g, "-") || "song";
-            a.download = `${safeName}.webm`;
-            a.click();
-            URL.revokeObjectURL(url);
-            audioCtx.close();
-            resolve();
-          };
-          requestAnimationFrame(tick);
+          processBatch();
         });
       } catch (e) {
         console.error("Video generation failed:", e);