Spaces:
Running
Running
Step 5: 配乐音效与混音
Browse files- .gitattributes +1 -0
- README.md +2 -1
- audio/soundtrack.mp3 +3 -0
- scripts/bake_audio.py +18 -0
- src/App.tsx +2 -0
- src/video/ADebug.tsx +24 -0
- src/video/TDebug.tsx +1 -1
- src/video/audio.ts +278 -0
- src/video/exporter.ts +69 -0
.gitattributes
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
audio/soundtrack.mp3 filter=lfs diff=lfs merge=lfs -text
|
README.md
CHANGED
|
@@ -60,7 +60,7 @@ pinned: false
|
|
| 60 |
2. ✅ 生成中文旁白逐句音频 + 时间轴清单(`timeline.json`)。
|
| 61 |
3. ✅ 程序化 3D 手部解剖模型:皮肤 / 骨骼 / 肌腱 / 腱鞘 / 关节骨架绑定 / 疼痛发光 / 器械与道具。
|
| 62 |
4. ✅ 分镜时间轴:11 镜头 + 结尾的相机运动、姿态、图层、字幕、HUD。
|
| 63 |
-
5.
|
| 64 |
6. ⏳ 播放器 UI(手机竖屏适配、全屏、进度条)+ MP4 本地导出(24/30/60fps)。
|
| 65 |
7. ⏳ 四平台文案(标题 / 话题 / 正文,严格字数校验)+ 封面生成与下载。
|
| 66 |
8. ⏳ 生产构建、检查,推送最终静态站点。
|
|
@@ -79,3 +79,4 @@ README.md ← 本文件(计划 + 进度)
|
|
| 79 |
- [x] Step 1:计划与大纲完成
|
| 80 |
- [x] Step 2:旁白 46 句逐句生成(zh-CN 神经网络语音,语速 +38%),自动裁剪静音并对齐时间轴,全片总长 **88.2 秒**(12 段)
|
| 81 |
- [x] Step 3-4:程序化 3D 手部解剖模型(皮肤透视 / 骨骼 / 肌腱 / 腱鞘、疼痛红光、原创器械与生活道具)+ 12 段分镜时间轴(镜头运动、分屏、字幕、HUD、超声示意、截面图),全时间轴逐帧验证无运行错误
|
|
|
|
|
|
| 60 |
2. ✅ 生成中文旁白逐句音频 + 时间轴清单(`timeline.json`)。
|
| 61 |
3. ✅ 程序化 3D 手部解剖模型:皮肤 / 骨骼 / 肌腱 / 腱鞘 / 关节骨架绑定 / 疼痛发光 / 器械与道具。
|
| 62 |
4. ✅ 分镜时间轴:11 镜头 + 结尾的相机运动、姿态、图层、字幕、HUD。
|
| 63 |
+
5. ✅ 程序化音乐与音效、旁白混音、ducking。
|
| 64 |
6. ⏳ 播放器 UI(手机竖屏适配、全屏、进度条)+ MP4 本地导出(24/30/60fps)。
|
| 65 |
7. ⏳ 四平台文案(标题 / 话题 / 正文,严格字数校验)+ 封面生成与下载。
|
| 66 |
8. ⏳ 生产构建、检查,推送最终静态站点。
|
|
|
|
| 79 |
- [x] Step 1:计划与大纲完成
|
| 80 |
- [x] Step 2:旁白 46 句逐句生成(zh-CN 神经网络语音,语速 +38%),自动裁剪静音并对齐时间轴,全片总长 **88.2 秒**(12 段)
|
| 81 |
- [x] Step 3-4:程序化 3D 手部解剖模型(皮肤透视 / 骨骼 / 肌腱 / 腱鞘、疼痛红光、原创器械与生活道具)+ 12 段分镜时间轴(镜头运动、分屏、字幕、HUD、超声示意、截面图),全时间轴逐帧验证无运行错误
|
| 82 |
+
- [x] Step 5:程序化配乐(紧张心跳前奏 → 小调解析 → 节奏推进 → 大调恢复)+ 10 类音效(疼痛脉冲、扫描、松解释放、提示音等,与画面事件逐帧对齐)+ 旁白自动闪避混音,离线渲染为 88.2 秒母带
|
audio/soundtrack.mp3
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:744e380816a7cd4b3090fd38a27e913328cf601083494e41235c69ed97cf20ba
|
| 3 |
+
size 2118528
|
scripts/bake_audio.py
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Render the procedural soundtrack in headless Chromium and bake it to frontend/public/audio/soundtrack.mp3"""
|
| 2 |
+
import asyncio, base64, lameenc
|
| 3 |
+
from playwright.async_api import async_playwright
|
| 4 |
+
URL = "http://localhost:5173/preview/production/ed5ba841-ed53-4eec-a8c5-93237d26b7ab/?adebug=1"
|
| 5 |
+
async def main():
|
| 6 |
+
async with async_playwright() as p:
|
| 7 |
+
b = await p.chromium.launch()
|
| 8 |
+
pg = await b.new_page()
|
| 9 |
+
pg.on("console", lambda m: print(m.text[:300]) if "INFO" in m.text else None)
|
| 10 |
+
await pg.goto(URL)
|
| 11 |
+
await pg.wait_for_function("window.__done === true", timeout=300000)
|
| 12 |
+
pcm = base64.b64decode(await pg.evaluate("window.__pcm"))
|
| 13 |
+
await b.close()
|
| 14 |
+
enc = lameenc.Encoder(); enc.set_bit_rate(192); enc.set_in_sample_rate(48000); enc.set_channels(2); enc.set_quality(2)
|
| 15 |
+
mp3 = enc.encode(pcm) + enc.flush()
|
| 16 |
+
open("frontend/public/audio/soundtrack.mp3", "wb").write(mp3)
|
| 17 |
+
print("bytes", len(mp3))
|
| 18 |
+
asyncio.run(main())
|
src/App.tsx
CHANGED
|
@@ -2,10 +2,12 @@ import { lazy, Suspense } from 'react'
|
|
| 2 |
|
| 3 |
const Debug = lazy(() => import('./video/Debug'))
|
| 4 |
const TDebug = lazy(() => import('./video/TDebug'))
|
|
|
|
| 5 |
|
| 6 |
export default function App() {
|
| 7 |
const q = typeof window !== 'undefined' ? new URLSearchParams(window.location.search) : null
|
| 8 |
if (q?.has('debug')) return <Suspense fallback={null}><Debug /></Suspense>
|
| 9 |
if (q?.has('tdebug')) return <Suspense fallback={null}><TDebug /></Suspense>
|
|
|
|
| 10 |
return <div style={{ color: '#fff' }}>loading</div>
|
| 11 |
}
|
|
|
|
| 2 |
|
| 3 |
const Debug = lazy(() => import('./video/Debug'))
|
| 4 |
const TDebug = lazy(() => import('./video/TDebug'))
|
| 5 |
+
const ADebug = lazy(() => import('./video/ADebug'))
|
| 6 |
|
| 7 |
export default function App() {
|
| 8 |
const q = typeof window !== 'undefined' ? new URLSearchParams(window.location.search) : null
|
| 9 |
if (q?.has('debug')) return <Suspense fallback={null}><Debug /></Suspense>
|
| 10 |
if (q?.has('tdebug')) return <Suspense fallback={null}><TDebug /></Suspense>
|
| 11 |
+
if (q?.has('adebug')) return <Suspense fallback={null}><ADebug /></Suspense>
|
| 12 |
return <div style={{ color: '#fff' }}>loading</div>
|
| 13 |
}
|
src/video/ADebug.tsx
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import { useEffect } from 'react'
|
| 2 |
+
import { getSoundtrack } from './audio'
|
| 3 |
+
import { SHOTS } from './scenes'
|
| 4 |
+
|
| 5 |
+
export default function ADebug() {
|
| 6 |
+
useEffect(() => {
|
| 7 |
+
const t0 = performance.now()
|
| 8 |
+
getSoundtrack(undefined, true).then(b => {
|
| 9 |
+
const d = b.getChannelData(0)
|
| 10 |
+
const rms = SHOTS.map(s => {
|
| 11 |
+
let a = 0, pk = 0; const i0 = Math.floor(s.start * b.sampleRate), i1 = Math.floor(s.end * b.sampleRate)
|
| 12 |
+
for (let i = i0; i < i1; i++) { a += d[i] * d[i]; pk = Math.max(pk, Math.abs(d[i])) }
|
| 13 |
+
return `${s.index}:${Math.sqrt(a / (i1 - i0)).toFixed(3)}/${pk.toFixed(2)}`
|
| 14 |
+
})
|
| 15 |
+
const L = b.getChannelData(0), R = b.getChannelData(1), pcm = new Int16Array(L.length * 2)
|
| 16 |
+
for (let i = 0; i < L.length; i++) { pcm[2 * i] = Math.max(-1, Math.min(1, L[i])) * 32767; pcm[2 * i + 1] = Math.max(-1, Math.min(1, R[i])) * 32767 }
|
| 17 |
+
const u8 = new Uint8Array(pcm.buffer); let bin = ''
|
| 18 |
+
for (let i = 0; i < u8.length; i += 0x8000) bin += String.fromCharCode(...u8.subarray(i, i + 0x8000))
|
| 19 |
+
;(window as unknown as { __pcm: string }).__pcm = btoa(bin)
|
| 20 |
+
console.log('INFO dur=' + b.duration.toFixed(2) + ' ms=' + (performance.now() - t0).toFixed(0) + ' ' + rms.join(' '))
|
| 21 |
+
}).catch(e => console.log('INFO ERROR ' + e)).finally(() => { (window as unknown as { __done: boolean }).__done = true })
|
| 22 |
+
}, [])
|
| 23 |
+
return null
|
| 24 |
+
}
|
src/video/TDebug.tsx
CHANGED
|
@@ -14,7 +14,7 @@ export default function TDebug() {
|
|
| 14 |
for (const t of ts) {
|
| 15 |
try {
|
| 16 |
const t0 = performance.now()
|
| 17 |
-
eng.render(frameAt(t), ctx, t
|
| 18 |
const ms = performance.now() - t0
|
| 19 |
const d = ctx.getImageData(0, 0, 540, 960).data
|
| 20 |
let sum = 0, red = 0, skin = 0
|
|
|
|
| 14 |
for (const t of ts) {
|
| 15 |
try {
|
| 16 |
const t0 = performance.now()
|
| 17 |
+
eng.render(frameAt(t), ctx, t)
|
| 18 |
const ms = performance.now() - t0
|
| 19 |
const d = ctx.getImageData(0, 0, 540, 960).data
|
| 20 |
let sum = 0, red = 0, skin = 0
|
src/video/audio.ts
ADDED
|
@@ -0,0 +1,278 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import { DURATION, LINES, SHOTS, soundCues, type CueKind } from './scenes'
|
| 2 |
+
|
| 3 |
+
export const SR = 48000
|
| 4 |
+
const BPM = 100
|
| 5 |
+
const BEAT = 60 / BPM
|
| 6 |
+
const BAR = BEAT * 4
|
| 7 |
+
|
| 8 |
+
type Ctx = OfflineAudioContext
|
| 9 |
+
|
| 10 |
+
const midi = (m: number) => 440 * Math.pow(2, (m - 69) / 12)
|
| 11 |
+
|
| 12 |
+
function noiseBuffer(ctx: Ctx, sec: number) {
|
| 13 |
+
const b = ctx.createBuffer(1, Math.ceil(sec * ctx.sampleRate), ctx.sampleRate)
|
| 14 |
+
const d = b.getChannelData(0)
|
| 15 |
+
let s = 1234567
|
| 16 |
+
for (let i = 0; i < d.length; i++) { s = (s * 16807) % 2147483647; d[i] = (s / 2147483647) * 2 - 1 }
|
| 17 |
+
return b
|
| 18 |
+
}
|
| 19 |
+
|
| 20 |
+
function impulse(ctx: Ctx, sec: number, decay: number) {
|
| 21 |
+
const n = Math.ceil(sec * ctx.sampleRate)
|
| 22 |
+
const b = ctx.createBuffer(2, n, ctx.sampleRate)
|
| 23 |
+
let s = 98765
|
| 24 |
+
for (let c = 0; c < 2; c++) {
|
| 25 |
+
const d = b.getChannelData(c)
|
| 26 |
+
for (let i = 0; i < n; i++) { s = (s * 16807) % 2147483647; d[i] = ((s / 2147483647) * 2 - 1) * Math.pow(1 - i / n, decay) }
|
| 27 |
+
}
|
| 28 |
+
return b
|
| 29 |
+
}
|
| 30 |
+
|
| 31 |
+
function env(g: GainNode, t: number, a: number, peak: number, hold: number, r: number) {
|
| 32 |
+
g.gain.setValueAtTime(0, t)
|
| 33 |
+
g.gain.linearRampToValueAtTime(peak, t + a)
|
| 34 |
+
g.gain.setValueAtTime(peak, t + a + hold)
|
| 35 |
+
g.gain.exponentialRampToValueAtTime(0.0001, t + a + hold + r)
|
| 36 |
+
}
|
| 37 |
+
|
| 38 |
+
/* ---------------- music ---------------- */
|
| 39 |
+
// sections: [start, end, chords(midi roots+voicing), mood]
|
| 40 |
+
interface Section { a: number; b: number; chords: number[][]; arp: boolean; kick: boolean; bright: number }
|
| 41 |
+
|
| 42 |
+
function sections(): Section[] {
|
| 43 |
+
const S = (i: number) => SHOTS[i].start
|
| 44 |
+
const minor = [[50, 57, 60, 64, 65], [46, 53, 57, 60, 62], [43, 50, 53, 58, 62], [45, 52, 57, 59, 64]] // Dm9 Bbmaj7 Gm9 Asus
|
| 45 |
+
const tense = [[38, 50, 53, 57], [38, 50, 52, 57]]
|
| 46 |
+
const major = [[41, 53, 57, 60, 64], [48, 55, 60, 64, 67], [50, 57, 60, 62, 65], [46, 53, 58, 62, 65]] // Fmaj7 C Dm Bb
|
| 47 |
+
return [
|
| 48 |
+
{ a: 0, b: S(1), chords: tense, arp: false, kick: false, bright: 0.2 },
|
| 49 |
+
{ a: S(1), b: S(4), chords: minor, arp: true, kick: false, bright: 0.45 },
|
| 50 |
+
{ a: S(4), b: S(8), chords: minor, arp: true, kick: true, bright: 0.6 },
|
| 51 |
+
{ a: S(8), b: DURATION, chords: major, arp: true, kick: true, bright: 0.85 },
|
| 52 |
+
]
|
| 53 |
+
}
|
| 54 |
+
|
| 55 |
+
function music(ctx: Ctx, out: AudioNode, rev: AudioNode) {
|
| 56 |
+
const secs = sections()
|
| 57 |
+
const nb = noiseBuffer(ctx, 1)
|
| 58 |
+
for (const s of secs) {
|
| 59 |
+
const nBars = Math.ceil((s.b - s.a) / BAR)
|
| 60 |
+
for (let k = 0; k < nBars; k++) {
|
| 61 |
+
const t0 = s.a + k * BAR
|
| 62 |
+
if (t0 >= s.b) break
|
| 63 |
+
const len = Math.min(BAR, s.b - t0) + 0.6
|
| 64 |
+
const ch = s.chords[k % s.chords.length]
|
| 65 |
+
// pad
|
| 66 |
+
ch.forEach((m, vi) => {
|
| 67 |
+
for (const det of [-7, 7]) {
|
| 68 |
+
const o = ctx.createOscillator(); o.type = vi === 0 ? 'sine' : 'sawtooth'
|
| 69 |
+
o.frequency.value = midi(vi === 0 ? m - 12 : m); o.detune.value = det
|
| 70 |
+
const f = ctx.createBiquadFilter(); f.type = 'lowpass'
|
| 71 |
+
f.frequency.setValueAtTime(500 + 900 * s.bright, t0)
|
| 72 |
+
f.frequency.linearRampToValueAtTime(900 + 1600 * s.bright, t0 + len * 0.6)
|
| 73 |
+
const g = ctx.createGain()
|
| 74 |
+
env(g, t0, 0.5, vi === 0 ? 0.11 : 0.028, len - 1.1, 0.9)
|
| 75 |
+
o.connect(f).connect(g); g.connect(out); g.connect(rev)
|
| 76 |
+
o.start(t0); o.stop(t0 + len + 0.3)
|
| 77 |
+
}
|
| 78 |
+
})
|
| 79 |
+
// arpeggio pluck (16ths in later sections, 8ths in middle)
|
| 80 |
+
if (s.arp) {
|
| 81 |
+
const step = s.kick ? BEAT / 4 : BEAT / 2
|
| 82 |
+
const notes = ch.slice(1).concat(ch.slice(1).map(x => x + 12))
|
| 83 |
+
const pat = [0, 2, 1, 3, 2, 4, 3, 5]
|
| 84 |
+
for (let i = 0; i * step < BAR && t0 + i * step < s.b; i++) {
|
| 85 |
+
const t = t0 + i * step
|
| 86 |
+
const m = notes[pat[i % pat.length] % notes.length]
|
| 87 |
+
const o = ctx.createOscillator(); o.type = 'triangle'; o.frequency.value = midi(m + 12)
|
| 88 |
+
const g = ctx.createGain(); env(g, t, 0.004, (i % 4 === 0 ? 0.05 : 0.032) * (0.7 + s.bright * 0.4), 0.02, step * 1.6)
|
| 89 |
+
const p = ctx.createStereoPanner(); p.pan.value = Math.sin(i * 1.3) * 0.5
|
| 90 |
+
o.connect(g).connect(p); p.connect(out); p.connect(rev)
|
| 91 |
+
o.start(t); o.stop(t + step * 2 + 0.1)
|
| 92 |
+
}
|
| 93 |
+
}
|
| 94 |
+
// heartbeat in the tense intro
|
| 95 |
+
if (!s.arp) {
|
| 96 |
+
for (let b = 0; b < 4; b += 2) for (const off of [0, 0.22]) {
|
| 97 |
+
const t = t0 + b * BEAT + off
|
| 98 |
+
if (t > s.b) continue
|
| 99 |
+
const o = ctx.createOscillator(); o.frequency.setValueAtTime(70, t); o.frequency.exponentialRampToValueAtTime(38, t + 0.18)
|
| 100 |
+
const g = ctx.createGain(); env(g, t, 0.005, off ? 0.22 : 0.32, 0.02, 0.25)
|
| 101 |
+
o.connect(g).connect(out); o.start(t); o.stop(t + 0.35)
|
| 102 |
+
}
|
| 103 |
+
}
|
| 104 |
+
// soft kick + hat
|
| 105 |
+
if (s.kick) {
|
| 106 |
+
for (let b = 0; b < 4; b++) {
|
| 107 |
+
const t = t0 + b * BEAT
|
| 108 |
+
if (t > s.b) continue
|
| 109 |
+
const o = ctx.createOscillator(); o.frequency.setValueAtTime(120, t); o.frequency.exponentialRampToValueAtTime(42, t + 0.12)
|
| 110 |
+
const g = ctx.createGain(); env(g, t, 0.003, 0.26, 0.01, 0.22)
|
| 111 |
+
o.connect(g).connect(out); o.start(t); o.stop(t + 0.3)
|
| 112 |
+
const h = ctx.createBufferSource(); h.buffer = nb
|
| 113 |
+
const hf = ctx.createBiquadFilter(); hf.type = 'highpass'; hf.frequency.value = 7000
|
| 114 |
+
const hg = ctx.createGain(); env(hg, t + BEAT / 2, 0.002, 0.035, 0, 0.05)
|
| 115 |
+
h.connect(hf).connect(hg).connect(out); h.start(t + BEAT / 2, ((k * 4 + b) % 5) * 0.1, 0.1)
|
| 116 |
+
}
|
| 117 |
+
}
|
| 118 |
+
}
|
| 119 |
+
}
|
| 120 |
+
// final resolve chord swell
|
| 121 |
+
const tf = SHOTS[SHOTS.length - 1].start + 0.4
|
| 122 |
+
;[41, 53, 60, 65, 69, 72].forEach(m => {
|
| 123 |
+
const o = ctx.createOscillator(); o.type = 'sine'; o.frequency.value = midi(m)
|
| 124 |
+
const g = ctx.createGain(); env(g, tf, 0.8, 0.05, 2, Math.max(0.5, DURATION - tf - 2.8))
|
| 125 |
+
o.connect(g); g.connect(out); g.connect(rev); o.start(tf); o.stop(DURATION)
|
| 126 |
+
})
|
| 127 |
+
}
|
| 128 |
+
|
| 129 |
+
/* ---------------- sfx ---------------- */
|
| 130 |
+
function sfx(ctx: Ctx, out: AudioNode, rev: AudioNode) {
|
| 131 |
+
const nb = noiseBuffer(ctx, 2)
|
| 132 |
+
const noise = (t: number, dur: number) => { const s = ctx.createBufferSource(); s.buffer = nb; s.start(t, 0, dur); return s }
|
| 133 |
+
const tone = (t: number, type: OscillatorType, f0: number, f1: number, dur: number) => {
|
| 134 |
+
const o = ctx.createOscillator(); o.type = type
|
| 135 |
+
o.frequency.setValueAtTime(f0, t); o.frequency.exponentialRampToValueAtTime(f1, t + dur)
|
| 136 |
+
o.start(t); o.stop(t + dur + 0.05); return o
|
| 137 |
+
}
|
| 138 |
+
const make: Record<CueKind, (t: number, v: number) => void> = {
|
| 139 |
+
whoosh: (t, v) => {
|
| 140 |
+
const s = noise(t, 0.6), f = ctx.createBiquadFilter(); f.type = 'bandpass'; f.Q.value = 2
|
| 141 |
+
f.frequency.setValueAtTime(300, t); f.frequency.exponentialRampToValueAtTime(3500, t + 0.45)
|
| 142 |
+
const g = ctx.createGain(); env(g, t, 0.25, 0.22 * v, 0, 0.3)
|
| 143 |
+
s.connect(f).connect(g); g.connect(out); g.connect(rev)
|
| 144 |
+
},
|
| 145 |
+
pain: (t, v) => {
|
| 146 |
+
const o = tone(t, 'sawtooth', 110, 55, 0.5), f = ctx.createBiquadFilter(); f.type = 'lowpass'; f.frequency.value = 600
|
| 147 |
+
const g = ctx.createGain(); env(g, t, 0.01, 0.2 * v, 0.05, 0.45)
|
| 148 |
+
o.connect(f).connect(g).connect(out)
|
| 149 |
+
const o2 = tone(t, 'sine', 80, 40, 0.4), g2 = ctx.createGain(); env(g2, t, 0.005, 0.4 * v, 0.02, 0.35)
|
| 150 |
+
o2.connect(g2).connect(out)
|
| 151 |
+
const o3 = tone(t + 0.02, 'sine', 1900, 1500, 0.35), g3 = ctx.createGain(); env(g3, t + 0.02, 0.01, 0.03 * v, 0, 0.3)
|
| 152 |
+
o3.connect(g3); g3.connect(out); g3.connect(rev)
|
| 153 |
+
},
|
| 154 |
+
tick: (t, v) => { const o = tone(t, 'sine', 2400, 2300, 0.04), g = ctx.createGain(); env(g, t, 0.002, 0.08 * v, 0, 0.04); o.connect(g).connect(out) },
|
| 155 |
+
scan: (t, v) => {
|
| 156 |
+
const o = tone(t, 'sine', 500, 1800, 0.8), g = ctx.createGain(); env(g, t, 0.05, 0.06 * v, 0.4, 0.35)
|
| 157 |
+
const lfo = ctx.createOscillator(); lfo.frequency.value = 18; const lg = ctx.createGain(); lg.gain.value = 0.03 * v
|
| 158 |
+
lfo.connect(lg).connect(g.gain); lfo.start(t); lfo.stop(t + 0.9)
|
| 159 |
+
o.connect(g); g.connect(out); g.connect(rev)
|
| 160 |
+
},
|
| 161 |
+
pop: (t, v) => { const o = tone(t, 'sine', 520, 980, 0.08), g = ctx.createGain(); env(g, t, 0.003, 0.12 * v, 0, 0.12); o.connect(g); g.connect(out); g.connect(rev) },
|
| 162 |
+
release: (t, v) => {
|
| 163 |
+
const s = noise(t, 1.2), f = ctx.createBiquadFilter(); f.type = 'lowpass'
|
| 164 |
+
f.frequency.setValueAtTime(2500, t); f.frequency.exponentialRampToValueAtTime(200, t + 1.1)
|
| 165 |
+
const g = ctx.createGain(); env(g, t, 0.02, 0.12 * v, 0.05, 1.0)
|
| 166 |
+
s.connect(f).connect(g); g.connect(out); g.connect(rev)
|
| 167 |
+
;[72, 76, 79, 84].forEach((m, i) => {
|
| 168 |
+
const o = tone(t + i * 0.07, 'sine', midi(m), midi(m) * 0.998, 1.4), gg = ctx.createGain()
|
| 169 |
+
env(gg, t + i * 0.07, 0.01, 0.05 * v, 0, 1.3); o.connect(gg); gg.connect(out); gg.connect(rev)
|
| 170 |
+
})
|
| 171 |
+
},
|
| 172 |
+
chime: (t, v) => {
|
| 173 |
+
for (const [r, a] of [[1, 1], [2.76, 0.5], [5.4, 0.25]]) {
|
| 174 |
+
const o = tone(t, 'sine', 880 * r, 880 * r, 1.6), g = ctx.createGain(); env(g, t, 0.003, 0.06 * v * a, 0, 1.5)
|
| 175 |
+
o.connect(g); g.connect(out); g.connect(rev)
|
| 176 |
+
}
|
| 177 |
+
},
|
| 178 |
+
thud: (t, v) => { const o = tone(t, 'sine', 90, 38, 0.3), g = ctx.createGain(); env(g, t, 0.003, 0.45 * v, 0.02, 0.3); o.connect(g).connect(out) },
|
| 179 |
+
click: (t, v) => {
|
| 180 |
+
const o = tone(t, 'square', 1600, 1200, 0.02), g = ctx.createGain(); env(g, t, 0.001, 0.04 * v, 0, 0.03); o.connect(g).connect(out)
|
| 181 |
+
const s = noise(t, 0.04), f = ctx.createBiquadFilter(); f.type = 'highpass'; f.frequency.value = 3000
|
| 182 |
+
const g2 = ctx.createGain(); env(g2, t, 0.001, 0.08 * v, 0, 0.03); s.connect(f).connect(g2).connect(out)
|
| 183 |
+
},
|
| 184 |
+
swipe: (t, v) => {
|
| 185 |
+
const s = noise(t, 0.3), f = ctx.createBiquadFilter(); f.type = 'highpass'
|
| 186 |
+
f.frequency.setValueAtTime(1200, t); f.frequency.linearRampToValueAtTime(4000, t + 0.25)
|
| 187 |
+
const g = ctx.createGain(); env(g, t, 0.06, 0.1 * v, 0, 0.2); s.connect(f).connect(g).connect(out)
|
| 188 |
+
},
|
| 189 |
+
}
|
| 190 |
+
for (const c of soundCues()) if (c.t >= 0 && c.t < DURATION) make[c.k](c.t, c.v ?? 1)
|
| 191 |
+
}
|
| 192 |
+
|
| 193 |
+
/* ---------------- narration ---------------- */
|
| 194 |
+
async function loadVoices(base: string, onProgress?: (p: number) => void) {
|
| 195 |
+
const tmp = new OfflineAudioContext(1, 1, SR)
|
| 196 |
+
let done = 0
|
| 197 |
+
return Promise.all(LINES.map(async l => {
|
| 198 |
+
const r = await fetch(base + 'audio/' + l.file)
|
| 199 |
+
if (!r.ok) throw new Error('voice ' + l.file + ' ' + r.status)
|
| 200 |
+
const buf = await tmp.decodeAudioData(await r.arrayBuffer())
|
| 201 |
+
onProgress?.(++done / LINES.length)
|
| 202 |
+
return { l, buf }
|
| 203 |
+
}))
|
| 204 |
+
}
|
| 205 |
+
|
| 206 |
+
let cached: Promise<AudioBuffer> | null = null
|
| 207 |
+
|
| 208 |
+
/** Full mixed soundtrack (narration + music + sfx), cached */
|
| 209 |
+
export function getSoundtrack(onProgress?: (p: number, label: string) => void, synth = false) {
|
| 210 |
+
if (!cached) cached = (synth ? build(onProgress) : baked(onProgress).catch(() => build(onProgress))).catch(e => { cached = null; throw e })
|
| 211 |
+
return cached
|
| 212 |
+
}
|
| 213 |
+
|
| 214 |
+
/** Pre-baked master (same mix, rendered once) — fast path */
|
| 215 |
+
async function baked(onProgress?: (p: number, label: string) => void) {
|
| 216 |
+
const base = (import.meta.env.BASE_URL || './').replace(/\/?$/, '/')
|
| 217 |
+
onProgress?.(0.1, '加载配乐')
|
| 218 |
+
const r = await fetch(base + 'audio/soundtrack.mp3')
|
| 219 |
+
if (!r.ok) throw new Error('soundtrack ' + r.status)
|
| 220 |
+
const ab = await r.arrayBuffer()
|
| 221 |
+
onProgress?.(0.6, '解码音频')
|
| 222 |
+
const ctx = new OfflineAudioContext(2, 1, SR)
|
| 223 |
+
const buf = await ctx.decodeAudioData(ab)
|
| 224 |
+
onProgress?.(1, '完成')
|
| 225 |
+
return buf
|
| 226 |
+
}
|
| 227 |
+
|
| 228 |
+
async function build(onProgress?: (p: number, label: string) => void) {
|
| 229 |
+
const base = (import.meta.env.BASE_URL || './').replace(/\/?$/, '/')
|
| 230 |
+
const voices = await loadVoices(base, p => onProgress?.(p * 0.6, '加载旁白'))
|
| 231 |
+
onProgress?.(0.65, '合成配乐')
|
| 232 |
+
const ctx = new OfflineAudioContext(2, Math.ceil(DURATION * SR), SR)
|
| 233 |
+
const master = ctx.createGain(); master.gain.value = 0.9
|
| 234 |
+
const comp = ctx.createDynamicsCompressor()
|
| 235 |
+
comp.threshold.value = -10; comp.knee.value = 6; comp.ratio.value = 8; comp.attack.value = 0.003; comp.release.value = 0.15
|
| 236 |
+
master.connect(comp).connect(ctx.destination)
|
| 237 |
+
master.gain.setValueAtTime(0.9, DURATION - 1.2); master.gain.linearRampToValueAtTime(0, DURATION)
|
| 238 |
+
|
| 239 |
+
const rev = ctx.createConvolver(); rev.buffer = impulse(ctx, 2.6, 3)
|
| 240 |
+
const revG = ctx.createGain(); revG.gain.value = 0.35
|
| 241 |
+
rev.connect(revG).connect(master)
|
| 242 |
+
|
| 243 |
+
// music bus with narration ducking
|
| 244 |
+
const mus = ctx.createGain(); mus.connect(master)
|
| 245 |
+
const musRev = ctx.createGain(); musRev.connect(rev)
|
| 246 |
+
const base0 = 0.85, duck = 0.38
|
| 247 |
+
for (const g of [mus.gain, musRev.gain]) {
|
| 248 |
+
g.setValueAtTime(0, 0); g.linearRampToValueAtTime(base0, 1.2)
|
| 249 |
+
const iv: [number, number][] = []
|
| 250 |
+
for (const { l } of voices) {
|
| 251 |
+
const a = l.start, b = l.start + l.dur, last = iv[iv.length - 1]
|
| 252 |
+
if (last && a - last[1] < 0.5) last[1] = Math.max(last[1], b); else iv.push([a, b])
|
| 253 |
+
}
|
| 254 |
+
for (const [a, b] of iv) {
|
| 255 |
+
g.setTargetAtTime(duck, Math.max(0, a - 0.12), 0.06)
|
| 256 |
+
g.setTargetAtTime(base0, b + 0.05, 0.25)
|
| 257 |
+
}
|
| 258 |
+
}
|
| 259 |
+
music(ctx, mus, musRev)
|
| 260 |
+
|
| 261 |
+
const fx = ctx.createGain(); fx.gain.value = 0.9; fx.connect(master)
|
| 262 |
+
sfx(ctx, fx, rev)
|
| 263 |
+
|
| 264 |
+
// narration with light EQ presence
|
| 265 |
+
const vo = ctx.createGain(); vo.gain.value = 1.15
|
| 266 |
+
const hp = ctx.createBiquadFilter(); hp.type = 'highpass'; hp.frequency.value = 90
|
| 267 |
+
const pres = ctx.createBiquadFilter(); pres.type = 'peaking'; pres.frequency.value = 3000; pres.gain.value = 2.5; pres.Q.value = 0.8
|
| 268 |
+
vo.connect(hp).connect(pres).connect(master)
|
| 269 |
+
const voRev = ctx.createGain(); voRev.gain.value = 0.06; vo.connect(voRev).connect(rev)
|
| 270 |
+
for (const { l, buf } of voices) {
|
| 271 |
+
const s = ctx.createBufferSource(); s.buffer = buf
|
| 272 |
+
s.connect(vo)
|
| 273 |
+
s.start(l.start, l.off, Math.min(l.dur, buf.duration - l.off))
|
| 274 |
+
}
|
| 275 |
+
const out = await ctx.startRendering()
|
| 276 |
+
onProgress?.(1, '完成')
|
| 277 |
+
return out
|
| 278 |
+
}
|
src/video/exporter.ts
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import {
|
| 2 |
+
Output, Mp4OutputFormat, BufferTarget, CanvasSource, AudioBufferSource,
|
| 3 |
+
getFirstEncodableVideoCodec, getFirstEncodableAudioCodec,
|
| 4 |
+
} from 'mediabunny'
|
| 5 |
+
import { Engine, LW, LH } from './engine'
|
| 6 |
+
import { frameAt, DURATION } from './scenes'
|
| 7 |
+
import { getSoundtrack } from './audio'
|
| 8 |
+
|
| 9 |
+
export type Fps = 24 | 30 | 60
|
| 10 |
+
|
| 11 |
+
export interface ExportProgress { phase: string; p: number; frame?: number; total?: number; eta?: number }
|
| 12 |
+
|
| 13 |
+
export async function exportMp4(fps: Fps, onProgress: (e: ExportProgress) => void, signal: AbortSignal): Promise<Blob> {
|
| 14 |
+
if (typeof VideoEncoder === 'undefined') throw new Error('当前浏览器不支持本地视频编码,请使用最新版 Chrome / Edge / Safari 17+')
|
| 15 |
+
const W = LW, H = LH
|
| 16 |
+
const bitrate = fps === 60 ? 16e6 : fps === 30 ? 12e6 : 10e6
|
| 17 |
+
const vcodec = await getFirstEncodableVideoCodec(['avc', 'hevc', 'vp9', 'av1'], { width: W, height: H, bitrate })
|
| 18 |
+
if (!vcodec) throw new Error('当前设备无法编码 1080×1920 视频')
|
| 19 |
+
onProgress({ phase: '准备音轨', p: 0 })
|
| 20 |
+
const sound = await getSoundtrack((p, l) => onProgress({ phase: l, p: p * 0.03 }))
|
| 21 |
+
const acodec = await getFirstEncodableAudioCodec(['aac', 'opus'], { numberOfChannels: 2, sampleRate: sound.sampleRate, bitrate: 192e3 })
|
| 22 |
+
|
| 23 |
+
const canvas = document.createElement('canvas'); canvas.width = W; canvas.height = H
|
| 24 |
+
const ctx = canvas.getContext('2d', { alpha: false })!
|
| 25 |
+
const eng = new Engine(); eng.setSize(W, H)
|
| 26 |
+
|
| 27 |
+
const output = new Output({ format: new Mp4OutputFormat({ fastStart: 'in-memory' }), target: new BufferTarget() })
|
| 28 |
+
const video = new CanvasSource(canvas, { codec: vcodec, bitrate, keyFrameInterval: 2 })
|
| 29 |
+
output.addVideoTrack(video, { frameRate: fps })
|
| 30 |
+
let audio: AudioBufferSource | null = null
|
| 31 |
+
if (acodec) { audio = new AudioBufferSource({ codec: acodec, bitrate: 192e3 }); output.addAudioTrack(audio) }
|
| 32 |
+
await output.start()
|
| 33 |
+
|
| 34 |
+
if (audio) {
|
| 35 |
+
// trim/pad to exact film length
|
| 36 |
+
const n = Math.round(DURATION * sound.sampleRate)
|
| 37 |
+
const ab = new AudioBuffer({ length: n, numberOfChannels: 2, sampleRate: sound.sampleRate })
|
| 38 |
+
for (let c = 0; c < 2; c++) ab.copyToChannel(sound.getChannelData(Math.min(c, sound.numberOfChannels - 1)).subarray(0, n), c)
|
| 39 |
+
await audio.add(ab); audio.close()
|
| 40 |
+
}
|
| 41 |
+
|
| 42 |
+
const total = Math.floor(DURATION * fps)
|
| 43 |
+
const t0 = performance.now()
|
| 44 |
+
try {
|
| 45 |
+
for (let i = 0; i < total; i++) {
|
| 46 |
+
if (signal.aborted) throw new DOMException('cancelled', 'AbortError')
|
| 47 |
+
const t = i / fps
|
| 48 |
+
eng.render(frameAt(t), ctx, t)
|
| 49 |
+
await video.add(t, 1 / fps)
|
| 50 |
+
if (i % 6 === 0 || i === total - 1) {
|
| 51 |
+
const el = (performance.now() - t0) / 1000
|
| 52 |
+
const p = (i + 1) / total
|
| 53 |
+
onProgress({ phase: '逐帧渲染编码', p: 0.03 + p * 0.95, frame: i + 1, total, eta: el / p * (1 - p) })
|
| 54 |
+
await new Promise(r => setTimeout(r, 0))
|
| 55 |
+
}
|
| 56 |
+
}
|
| 57 |
+
video.close()
|
| 58 |
+
onProgress({ phase: '封装 MP4', p: 0.99 })
|
| 59 |
+
await output.finalize()
|
| 60 |
+
} catch (e) {
|
| 61 |
+
await output.cancel().catch(() => {})
|
| 62 |
+
eng.renderer.dispose()
|
| 63 |
+
throw e
|
| 64 |
+
}
|
| 65 |
+
eng.renderer.dispose()
|
| 66 |
+
const buf = (output.target as BufferTarget).buffer!
|
| 67 |
+
onProgress({ phase: '完成', p: 1 })
|
| 68 |
+
return new Blob([buf], { type: 'video/mp4' })
|
| 69 |
+
}
|