File size: 3,344 Bytes
550ce03
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
"""Bounded microbatch queue: combine independent callers into real ONNX batches."""
from concurrent.futures import Future
import math
from queue import Queue, Empty, Full
import threading
import time


class BatchQueue:
    def __init__(self, synthesizer, max_wait_ms=2, max_pending=256):
        if max_wait_ms < 0 or max_pending < 1:
            raise ValueError("Queue settings must be nonnegative with positive capacity")
        self.synthesizer = synthesizer
        self.max_wait = max_wait_ms / 1000
        self.queue = Queue(maxsize=max_pending)
        self.closed = False
        self.lock = threading.Lock()
        self.worker = threading.Thread(target=self._work, name="little-tts-batcher", daemon=True)
        self.worker.start()

    def submit(self, text, voice="92", speed=1.0, seed=0):
        if not isinstance(text, str) or not text.strip():
            raise ValueError("Each request needs nonempty text")
        if voice not in self.synthesizer.voices:
            raise ValueError("Unknown voice")
        if not isinstance(speed, (float, int)) or not math.isfinite(speed) or not .5 <= speed <= 2:
            raise ValueError("Speed must be between .5 and 2")
        if not isinstance(seed, int) or seed < 0:
            raise ValueError("Seed must be a nonnegative integer")
        future = Future()
        with self.lock:
            if self.closed:
                raise RuntimeError("Batch queue is closed")
            try:
                self.queue.put_nowait((future, text, voice, speed, seed))
            except Full:
                raise RuntimeError("Batch queue is full; retry later") from None
        return future

    def _work(self):
        stopping = False
        while not stopping:
            first = self.queue.get()
            if first is None:
                break
            pending = [first]
            deadline = time.monotonic() + self.max_wait
            while len(pending) < self.synthesizer.max_batch:
                remaining = deadline - time.monotonic()
                try:
                    item = self.queue.get(timeout=max(0, remaining))
                except Empty:
                    break
                if item is None:
                    stopping = True
                    break
                pending.append(item)
            active = [r for r in pending if r[0].set_running_or_notify_cancel()]
            if not active:
                continue
            try:
                results = self.synthesizer.speak_batch([r[1] for r in active], voices=[r[2] for r in active],
                                                       speeds=[r[3] for r in active], seeds=[r[4] for r in active])
                if len(results) != len(active):
                    raise RuntimeError("Inference returned an unexpected result count")
            except Exception as error:
                for request in active:
                    request[0].set_exception(error)
            else:
                for request, result in zip(active, results):
                    request[0].set_result(result)

    def close(self):
        with self.lock:
            if self.closed:
                return
            self.closed = True
        self.queue.put(None)
        self.worker.join()

    def __enter__(self): return self
    def __exit__(self, *args): self.close()