File size: 5,323 Bytes
83a951c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
"""Offline tests of the live ASR adapters against recorded-format provider messages (tests/fixtures, made-up content).

    python tests/test_adapters.py            # or: pytest tests/

1. call sequence: a recording fake stream checks the exact push_words / endpoint calls each adapter makes;
2. end to end: the same messages drive a real ArcTagger stream; checks words, speakers, utterances and JSON-able output.
"""
import json, os, sys

HERE = os.path.dirname(os.path.abspath(__file__)); ROOT = os.path.dirname(HERE); sys.path.insert(0, ROOT)
from arctag.asr import DeepgramAdapter, SpeechmaticsAdapter, summarize   # noqa: E402

FIX = os.path.join(HERE, "fixtures")
U1 = ["Hi,", "this", "is", "Dana", "from", "Acme", "Telecom."]
U2 = ["How", "can", "I", "help", "you", "today?"]
U3 = ["Yeah,", "my", "name", "is", "John", "Patel."]
U4 = ["And", "my", "number", "is", "555-213-4489."]
U5 = ["Thanks,", "John.", "I'll", "send", "a", "text", "to", "that", "number."]
ALL = U1 + U2 + U3 + U4 + U5


def P(words, final, k):
    return ("push_words", {"words": words, "final": final, "revise_from": k})


E = ("endpoint", {})
EXPECT = {
    "deepgram": [P(["hi", "this", "is"], False, 0), P(["hi", "this", "is", "dana", "from"], False, 0), P(U1, True, 0),
                 P(["how", "can", "i"], False, 7), P(U2, True, 7), E,
                 P(["yeah", "my", "name"], False, 13), P(["yeah", "my", "name", "is", "john"], False, 13), P(U3, True, 13),
                 P(["and", "my", "number", "is", "five", "five", "five"], False, 19),
                 P(U4, True, 19), E,                      # UtteranceEnd arrived with interim words pending: endpoint after the final
                 P(["thanks", "john"], False, 24), P(U5, True, 24), E],
    "speechmatics": [P(["Hi", "this"], False, 0), P(["Hi,", "this", "is", "Dana"], False, 0), P(U1, True, 0),
                     P(["How", "can", "I"], False, 7), P(U2, True, 7), E,
                     P(["Yeah", "my", "name"], False, 13), P(U3, True, 13),
                     P(["And", "my", "number", "is", "555"], False, 19),
                     P(U4, True, 19), E,                  # EndOfUtterance arrived with partial words pending
                     P(["Thanks", "John"], False, 24), P(U5, True, 24), E],
}
ADAPTERS = {"deepgram": (DeepgramAdapter, "deepgram_live.jsonl"), "speechmatics": (SpeechmaticsAdapter, "speechmatics_rt.jsonl")}
SPK = {"deepgram": ("S0", "S1"), "speechmatics": ("S1", "S2")}


def messages(name):
    return [json.loads(l) for l in open(os.path.join(FIX, ADAPTERS[name][1])) if l.strip()]


class FakeStream:
    """records calls; keeps the word list like the real stream does"""
    def __init__(self):
        self.words = []; self.n_endpoints = 0; self.spk = []

    def push_words(self, words, final=True, revise_from=None, endpoint=False):
        k = len(self.words) if revise_from is None else revise_from
        assert 0 <= k <= len(self.words)
        self.words = self.words[:k] + [dict(w, final=final) for w in words]
        return {"n_words": len(self.words), "spans": [], "events": [], "utterances": [], "running": {}}

    def endpoint(self):
        assert all(w["final"] for w in self.words), "endpoint with non-final words"
        self.n_endpoints += 1
        return {"n_words": len(self.words), "spans": [], "events": [], "utterances": [{}], "running": {}}

    def redacted_text(self):
        return " ".join(w["w"] for w in self.words)


def check_sequence(name):
    cls, _ = ADAPTERS[name]
    fs = FakeStream(); ad = cls(fs)
    for m in messages(name):
        ad(m)
    assert ad.calls == EXPECT[name], json.dumps(ad.calls, indent=0)
    assert [w["w"] for w in fs.words] == ALL and all(w["final"] for w in fs.words)
    spk = [w.get("spk") for w in fs.words]
    a, b = SPK[name]
    assert spk == [a] * 13 + [b] * 11 + [a] * 9, spk
    assert fs.n_endpoints == 3
    return len(ad.calls)


def check_end_to_end(name, tg):
    cls, _ = ADAPTERS[name]
    st = tg.stream(domain="talk"); ad = cls(st); lines = []
    for m in messages(name):
        for up in ad(m):
            lines.append(json.dumps(summarize(up, st)))          # must be JSON-serialisable
    res = st.finalize()
    assert res["words"] == ALL, res["words"]
    assert [(u["start"], u["end"]) for u in res["utterances"]] == [(0, 13), (13, 24), (24, 33)], res["utterances"]
    assert [w.get("spk") for w in st.s.words][:2] == [SPK[name][0]] * 2
    return {"updates": len(lines), "redacted_text": res["redacted_text"],
            "spans": [(s["tag"], s["text"]) for s in res["spans"]],
            "acts": [u["act"] for u in res["utterances"]], "roles": [u["role"] for u in res["utterances"]]}


def test_deepgram_sequence():
    check_sequence("deepgram")


def test_speechmatics_sequence():
    check_sequence("speechmatics")


def test_end_to_end():
    from arctag import ArcTagger
    tg = ArcTagger.from_pretrained(ROOT)
    for name in ADAPTERS:
        check_end_to_end(name, tg)


if __name__ == "__main__":
    for name in ADAPTERS:
        print(f"{name}: call sequence OK ({check_sequence(name)} calls)")
    import torch
    torch.set_num_threads(1)
    from arctag import ArcTagger
    tg = ArcTagger.from_pretrained(ROOT)
    for name in ADAPTERS:
        print(f"{name}: end to end OK", json.dumps(check_end_to_end(name, tg)))