Spaces:
Running
Running
NeonClary commited on
Commit ·
dc20b43
1
Parent(s): a2abb02
feat: add skip-back button for TTS playback (replay current or previous sentence)
Browse files- frontend/src/App.jsx +68 -5
frontend/src/App.jsx
CHANGED
|
@@ -17,6 +17,7 @@ import {
|
|
| 17 |
Square,
|
| 18 |
Share2,
|
| 19 |
Trash2,
|
|
|
|
| 20 |
Volume2,
|
| 21 |
} from 'lucide-react'
|
| 22 |
|
|
@@ -270,6 +271,17 @@ function AssistantSearchBar({ content, show, speak }) {
|
|
| 270 |
<Volume2 size={14} aria-hidden />
|
| 271 |
)}
|
| 272 |
</button>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 273 |
{(speak.playing || speak.paused || (speak.loading && speak.showStop)) && (
|
| 274 |
<button
|
| 275 |
type="button"
|
|
@@ -394,6 +406,8 @@ export default function App() {
|
|
| 394 |
streamingRef.current = streaming
|
| 395 |
const ttsContentResolverRef = useRef(null)
|
| 396 |
const ttsPlaybackActiveRef = useRef(false)
|
|
|
|
|
|
|
| 397 |
const streamingWasRef = useRef(false)
|
| 398 |
const mediaRecorderRef = useRef(null)
|
| 399 |
const audioChunksRef = useRef([])
|
|
@@ -557,6 +571,8 @@ export default function App() {
|
|
| 557 |
|
| 558 |
const stopTts = useCallback(() => {
|
| 559 |
ttsSessionRef.current += 1
|
|
|
|
|
|
|
| 560 |
if (audioRef.current) {
|
| 561 |
audioRef.current.pause()
|
| 562 |
audioRef.current.src = ''
|
|
@@ -590,6 +606,20 @@ export default function App() {
|
|
| 590 |
}
|
| 591 |
}, [])
|
| 592 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 593 |
const playAudioUrlUntilDone = useCallback((url, session) => {
|
| 594 |
return new Promise((resolve) => {
|
| 595 |
if (session !== ttsSessionRef.current) {
|
|
@@ -611,6 +641,10 @@ export default function App() {
|
|
| 611 |
}
|
| 612 |
pollAbort = setInterval(() => {
|
| 613 |
if (session !== ttsSessionRef.current) finish()
|
|
|
|
|
|
|
|
|
|
|
|
|
| 614 |
}, 120)
|
| 615 |
audio.onended = () => {
|
| 616 |
finish()
|
|
@@ -675,6 +709,16 @@ export default function App() {
|
|
| 675 |
let anyPlayed = false
|
| 676 |
let lastErr = null
|
| 677 |
const inFlight = new Map()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 678 |
|
| 679 |
const getSentencesAndLimit = () => {
|
| 680 |
const live = streamingRef.current
|
|
@@ -692,14 +736,19 @@ export default function App() {
|
|
| 692 |
|
| 693 |
if (playedCount < limit) {
|
| 694 |
for (let ahead = playedCount; ahead < Math.min(playedCount + LOOKAHEAD, limit); ahead++) {
|
| 695 |
-
if (!inFlight.has(ahead)) {
|
| 696 |
inFlight.set(ahead, fetchTtsAudio(sentences[ahead]))
|
| 697 |
}
|
| 698 |
}
|
| 699 |
|
| 700 |
-
|
| 701 |
-
|
| 702 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 703 |
|
| 704 |
if (session !== ttsSessionRef.current) return
|
| 705 |
|
|
@@ -714,8 +763,20 @@ export default function App() {
|
|
| 714 |
setTtsPlayingIndex(index)
|
| 715 |
setTtsPaused(false)
|
| 716 |
|
|
|
|
|
|
|
|
|
|
| 717 |
await playAudioUrlUntilDone(result.url, session)
|
| 718 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 719 |
if (ttsBlobUrlRef.current === result.url) {
|
| 720 |
ttsBlobUrlRef.current = null
|
| 721 |
audioRef.current = null
|
|
@@ -751,6 +812,7 @@ export default function App() {
|
|
| 751 |
setTtsPaused(false)
|
| 752 |
} finally {
|
| 753 |
ttsPlaybackActiveRef.current = false
|
|
|
|
| 754 |
}
|
| 755 |
},
|
| 756 |
[stopTts, playAudioUrlUntilDone, fetchTtsAudio],
|
|
@@ -1412,6 +1474,7 @@ export default function App() {
|
|
| 1412 |
onReadAloud: () => playTtsForIndex(i, m.content),
|
| 1413 |
onPause: pauseTts,
|
| 1414 |
onResume: resumeTts,
|
|
|
|
| 1415 |
onStopReading: stopTts,
|
| 1416 |
}}
|
| 1417 |
/>
|
|
|
|
| 17 |
Square,
|
| 18 |
Share2,
|
| 19 |
Trash2,
|
| 20 |
+
SkipBack,
|
| 21 |
Volume2,
|
| 22 |
} from 'lucide-react'
|
| 23 |
|
|
|
|
| 271 |
<Volume2 size={14} aria-hidden />
|
| 272 |
)}
|
| 273 |
</button>
|
| 274 |
+
{(speak.playing || speak.paused) && (
|
| 275 |
+
<button
|
| 276 |
+
type="button"
|
| 277 |
+
className="aj-msg-search-btn"
|
| 278 |
+
onClick={speak.onReplay}
|
| 279 |
+
data-tip="Skip back"
|
| 280 |
+
aria-label="Skip back"
|
| 281 |
+
>
|
| 282 |
+
<SkipBack size={14} aria-hidden />
|
| 283 |
+
</button>
|
| 284 |
+
)}
|
| 285 |
{(speak.playing || speak.paused || (speak.loading && speak.showStop)) && (
|
| 286 |
<button
|
| 287 |
type="button"
|
|
|
|
| 406 |
streamingRef.current = streaming
|
| 407 |
const ttsContentResolverRef = useRef(null)
|
| 408 |
const ttsPlaybackActiveRef = useRef(false)
|
| 409 |
+
const ttsSkipRef = useRef(false)
|
| 410 |
+
const ttsReplayDeltaRef = useRef(null)
|
| 411 |
const streamingWasRef = useRef(false)
|
| 412 |
const mediaRecorderRef = useRef(null)
|
| 413 |
const audioChunksRef = useRef([])
|
|
|
|
| 571 |
|
| 572 |
const stopTts = useCallback(() => {
|
| 573 |
ttsSessionRef.current += 1
|
| 574 |
+
ttsSkipRef.current = false
|
| 575 |
+
ttsReplayDeltaRef.current = null
|
| 576 |
if (audioRef.current) {
|
| 577 |
audioRef.current.pause()
|
| 578 |
audioRef.current.src = ''
|
|
|
|
| 606 |
}
|
| 607 |
}, [])
|
| 608 |
|
| 609 |
+
const replayTts = useCallback(() => {
|
| 610 |
+
const a = audioRef.current
|
| 611 |
+
if (!a) return
|
| 612 |
+
const EARLY_THRESHOLD = 1.0
|
| 613 |
+
if (a.currentTime > EARLY_THRESHOLD) {
|
| 614 |
+
a.currentTime = 0
|
| 615 |
+
if (a.paused) a.play().catch(() => {})
|
| 616 |
+
setTtsPaused(false)
|
| 617 |
+
} else {
|
| 618 |
+
ttsReplayDeltaRef.current = -1
|
| 619 |
+
ttsSkipRef.current = true
|
| 620 |
+
}
|
| 621 |
+
}, [])
|
| 622 |
+
|
| 623 |
const playAudioUrlUntilDone = useCallback((url, session) => {
|
| 624 |
return new Promise((resolve) => {
|
| 625 |
if (session !== ttsSessionRef.current) {
|
|
|
|
| 641 |
}
|
| 642 |
pollAbort = setInterval(() => {
|
| 643 |
if (session !== ttsSessionRef.current) finish()
|
| 644 |
+
if (ttsSkipRef.current) {
|
| 645 |
+
audio.pause()
|
| 646 |
+
finish()
|
| 647 |
+
}
|
| 648 |
}, 120)
|
| 649 |
audio.onended = () => {
|
| 650 |
finish()
|
|
|
|
| 709 |
let anyPlayed = false
|
| 710 |
let lastErr = null
|
| 711 |
const inFlight = new Map()
|
| 712 |
+
const recentUrls = new Map()
|
| 713 |
+
|
| 714 |
+
const revokeRecent = (keepFrom = Infinity) => {
|
| 715 |
+
for (const [idx, u] of recentUrls) {
|
| 716 |
+
if (idx < keepFrom) {
|
| 717 |
+
URL.revokeObjectURL(u)
|
| 718 |
+
recentUrls.delete(idx)
|
| 719 |
+
}
|
| 720 |
+
}
|
| 721 |
+
}
|
| 722 |
|
| 723 |
const getSentencesAndLimit = () => {
|
| 724 |
const live = streamingRef.current
|
|
|
|
| 736 |
|
| 737 |
if (playedCount < limit) {
|
| 738 |
for (let ahead = playedCount; ahead < Math.min(playedCount + LOOKAHEAD, limit); ahead++) {
|
| 739 |
+
if (!inFlight.has(ahead) && !recentUrls.has(ahead)) {
|
| 740 |
inFlight.set(ahead, fetchTtsAudio(sentences[ahead]))
|
| 741 |
}
|
| 742 |
}
|
| 743 |
|
| 744 |
+
let result
|
| 745 |
+
if (recentUrls.has(playedCount)) {
|
| 746 |
+
result = { url: recentUrls.get(playedCount) }
|
| 747 |
+
} else {
|
| 748 |
+
if (!anyPlayed) setTtsLoadingIndex(index)
|
| 749 |
+
result = await (inFlight.get(playedCount) || fetchTtsAudio(sentences[playedCount]))
|
| 750 |
+
inFlight.delete(playedCount)
|
| 751 |
+
}
|
| 752 |
|
| 753 |
if (session !== ttsSessionRef.current) return
|
| 754 |
|
|
|
|
| 763 |
setTtsPlayingIndex(index)
|
| 764 |
setTtsPaused(false)
|
| 765 |
|
| 766 |
+
recentUrls.set(playedCount, result.url)
|
| 767 |
+
revokeRecent(playedCount - 2)
|
| 768 |
+
|
| 769 |
await playAudioUrlUntilDone(result.url, session)
|
| 770 |
+
|
| 771 |
+
if (ttsSkipRef.current) {
|
| 772 |
+
ttsSkipRef.current = false
|
| 773 |
+
const delta = ttsReplayDeltaRef.current ?? 0
|
| 774 |
+
ttsReplayDeltaRef.current = null
|
| 775 |
+
playedCount = Math.max(0, playedCount + delta)
|
| 776 |
+
setTtsPaused(false)
|
| 777 |
+
continue
|
| 778 |
+
}
|
| 779 |
+
|
| 780 |
if (ttsBlobUrlRef.current === result.url) {
|
| 781 |
ttsBlobUrlRef.current = null
|
| 782 |
audioRef.current = null
|
|
|
|
| 812 |
setTtsPaused(false)
|
| 813 |
} finally {
|
| 814 |
ttsPlaybackActiveRef.current = false
|
| 815 |
+
revokeRecent()
|
| 816 |
}
|
| 817 |
},
|
| 818 |
[stopTts, playAudioUrlUntilDone, fetchTtsAudio],
|
|
|
|
| 1474 |
onReadAloud: () => playTtsForIndex(i, m.content),
|
| 1475 |
onPause: pauseTts,
|
| 1476 |
onResume: resumeTts,
|
| 1477 |
+
onReplay: replayTts,
|
| 1478 |
onStopReading: stopTts,
|
| 1479 |
}}
|
| 1480 |
/>
|