NeonClary commited on
Commit
dc20b43
·
1 Parent(s): a2abb02

feat: add skip-back button for TTS playback (replay current or previous sentence)

Browse files
Files changed (1) hide show
  1. frontend/src/App.jsx +68 -5
frontend/src/App.jsx CHANGED
@@ -17,6 +17,7 @@ import {
17
  Square,
18
  Share2,
19
  Trash2,
 
20
  Volume2,
21
  } from 'lucide-react'
22
 
@@ -270,6 +271,17 @@ function AssistantSearchBar({ content, show, speak }) {
270
  <Volume2 size={14} aria-hidden />
271
  )}
272
  </button>
 
 
 
 
 
 
 
 
 
 
 
273
  {(speak.playing || speak.paused || (speak.loading && speak.showStop)) && (
274
  <button
275
  type="button"
@@ -394,6 +406,8 @@ export default function App() {
394
  streamingRef.current = streaming
395
  const ttsContentResolverRef = useRef(null)
396
  const ttsPlaybackActiveRef = useRef(false)
 
 
397
  const streamingWasRef = useRef(false)
398
  const mediaRecorderRef = useRef(null)
399
  const audioChunksRef = useRef([])
@@ -557,6 +571,8 @@ export default function App() {
557
 
558
  const stopTts = useCallback(() => {
559
  ttsSessionRef.current += 1
 
 
560
  if (audioRef.current) {
561
  audioRef.current.pause()
562
  audioRef.current.src = ''
@@ -590,6 +606,20 @@ export default function App() {
590
  }
591
  }, [])
592
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
593
  const playAudioUrlUntilDone = useCallback((url, session) => {
594
  return new Promise((resolve) => {
595
  if (session !== ttsSessionRef.current) {
@@ -611,6 +641,10 @@ export default function App() {
611
  }
612
  pollAbort = setInterval(() => {
613
  if (session !== ttsSessionRef.current) finish()
 
 
 
 
614
  }, 120)
615
  audio.onended = () => {
616
  finish()
@@ -675,6 +709,16 @@ export default function App() {
675
  let anyPlayed = false
676
  let lastErr = null
677
  const inFlight = new Map()
 
 
 
 
 
 
 
 
 
 
678
 
679
  const getSentencesAndLimit = () => {
680
  const live = streamingRef.current
@@ -692,14 +736,19 @@ export default function App() {
692
 
693
  if (playedCount < limit) {
694
  for (let ahead = playedCount; ahead < Math.min(playedCount + LOOKAHEAD, limit); ahead++) {
695
- if (!inFlight.has(ahead)) {
696
  inFlight.set(ahead, fetchTtsAudio(sentences[ahead]))
697
  }
698
  }
699
 
700
- if (!anyPlayed) setTtsLoadingIndex(index)
701
- const result = await (inFlight.get(playedCount) || fetchTtsAudio(sentences[playedCount]))
702
- inFlight.delete(playedCount)
 
 
 
 
 
703
 
704
  if (session !== ttsSessionRef.current) return
705
 
@@ -714,8 +763,20 @@ export default function App() {
714
  setTtsPlayingIndex(index)
715
  setTtsPaused(false)
716
 
 
 
 
717
  await playAudioUrlUntilDone(result.url, session)
718
- URL.revokeObjectURL(result.url)
 
 
 
 
 
 
 
 
 
719
  if (ttsBlobUrlRef.current === result.url) {
720
  ttsBlobUrlRef.current = null
721
  audioRef.current = null
@@ -751,6 +812,7 @@ export default function App() {
751
  setTtsPaused(false)
752
  } finally {
753
  ttsPlaybackActiveRef.current = false
 
754
  }
755
  },
756
  [stopTts, playAudioUrlUntilDone, fetchTtsAudio],
@@ -1412,6 +1474,7 @@ export default function App() {
1412
  onReadAloud: () => playTtsForIndex(i, m.content),
1413
  onPause: pauseTts,
1414
  onResume: resumeTts,
 
1415
  onStopReading: stopTts,
1416
  }}
1417
  />
 
17
  Square,
18
  Share2,
19
  Trash2,
20
+ SkipBack,
21
  Volume2,
22
  } from 'lucide-react'
23
 
 
271
  <Volume2 size={14} aria-hidden />
272
  )}
273
  </button>
274
+ {(speak.playing || speak.paused) && (
275
+ <button
276
+ type="button"
277
+ className="aj-msg-search-btn"
278
+ onClick={speak.onReplay}
279
+ data-tip="Skip back"
280
+ aria-label="Skip back"
281
+ >
282
+ <SkipBack size={14} aria-hidden />
283
+ </button>
284
+ )}
285
  {(speak.playing || speak.paused || (speak.loading && speak.showStop)) && (
286
  <button
287
  type="button"
 
406
  streamingRef.current = streaming
407
  const ttsContentResolverRef = useRef(null)
408
  const ttsPlaybackActiveRef = useRef(false)
409
+ const ttsSkipRef = useRef(false)
410
+ const ttsReplayDeltaRef = useRef(null)
411
  const streamingWasRef = useRef(false)
412
  const mediaRecorderRef = useRef(null)
413
  const audioChunksRef = useRef([])
 
571
 
572
  const stopTts = useCallback(() => {
573
  ttsSessionRef.current += 1
574
+ ttsSkipRef.current = false
575
+ ttsReplayDeltaRef.current = null
576
  if (audioRef.current) {
577
  audioRef.current.pause()
578
  audioRef.current.src = ''
 
606
  }
607
  }, [])
608
 
609
+ const replayTts = useCallback(() => {
610
+ const a = audioRef.current
611
+ if (!a) return
612
+ const EARLY_THRESHOLD = 1.0
613
+ if (a.currentTime > EARLY_THRESHOLD) {
614
+ a.currentTime = 0
615
+ if (a.paused) a.play().catch(() => {})
616
+ setTtsPaused(false)
617
+ } else {
618
+ ttsReplayDeltaRef.current = -1
619
+ ttsSkipRef.current = true
620
+ }
621
+ }, [])
622
+
623
  const playAudioUrlUntilDone = useCallback((url, session) => {
624
  return new Promise((resolve) => {
625
  if (session !== ttsSessionRef.current) {
 
641
  }
642
  pollAbort = setInterval(() => {
643
  if (session !== ttsSessionRef.current) finish()
644
+ if (ttsSkipRef.current) {
645
+ audio.pause()
646
+ finish()
647
+ }
648
  }, 120)
649
  audio.onended = () => {
650
  finish()
 
709
  let anyPlayed = false
710
  let lastErr = null
711
  const inFlight = new Map()
712
+ const recentUrls = new Map()
713
+
714
+ const revokeRecent = (keepFrom = Infinity) => {
715
+ for (const [idx, u] of recentUrls) {
716
+ if (idx < keepFrom) {
717
+ URL.revokeObjectURL(u)
718
+ recentUrls.delete(idx)
719
+ }
720
+ }
721
+ }
722
 
723
  const getSentencesAndLimit = () => {
724
  const live = streamingRef.current
 
736
 
737
  if (playedCount < limit) {
738
  for (let ahead = playedCount; ahead < Math.min(playedCount + LOOKAHEAD, limit); ahead++) {
739
+ if (!inFlight.has(ahead) && !recentUrls.has(ahead)) {
740
  inFlight.set(ahead, fetchTtsAudio(sentences[ahead]))
741
  }
742
  }
743
 
744
+ let result
745
+ if (recentUrls.has(playedCount)) {
746
+ result = { url: recentUrls.get(playedCount) }
747
+ } else {
748
+ if (!anyPlayed) setTtsLoadingIndex(index)
749
+ result = await (inFlight.get(playedCount) || fetchTtsAudio(sentences[playedCount]))
750
+ inFlight.delete(playedCount)
751
+ }
752
 
753
  if (session !== ttsSessionRef.current) return
754
 
 
763
  setTtsPlayingIndex(index)
764
  setTtsPaused(false)
765
 
766
+ recentUrls.set(playedCount, result.url)
767
+ revokeRecent(playedCount - 2)
768
+
769
  await playAudioUrlUntilDone(result.url, session)
770
+
771
+ if (ttsSkipRef.current) {
772
+ ttsSkipRef.current = false
773
+ const delta = ttsReplayDeltaRef.current ?? 0
774
+ ttsReplayDeltaRef.current = null
775
+ playedCount = Math.max(0, playedCount + delta)
776
+ setTtsPaused(false)
777
+ continue
778
+ }
779
+
780
  if (ttsBlobUrlRef.current === result.url) {
781
  ttsBlobUrlRef.current = null
782
  audioRef.current = null
 
812
  setTtsPaused(false)
813
  } finally {
814
  ttsPlaybackActiveRef.current = false
815
+ revokeRecent()
816
  }
817
  },
818
  [stopTts, playAudioUrlUntilDone, fetchTtsAudio],
 
1474
  onReadAloud: () => playTtsForIndex(i, m.content),
1475
  onPause: pauseTts,
1476
  onResume: resumeTts,
1477
+ onReplay: replayTts,
1478
  onStopReading: stopTts,
1479
  }}
1480
  />