arc-tagger / tests /test_adapters.py
ksingla025's picture
Arc Tagger v0.4.0
83a951c
Raw History Blame Contribute Delete
5.32 kB
"""Offline tests of the live ASR adapters against recorded-format provider messages (tests/fixtures, made-up content).
python tests/test_adapters.py # or: pytest tests/
1. call sequence: a recording fake stream checks the exact push_words / endpoint calls each adapter makes;
2. end to end: the same messages drive a real ArcTagger stream; checks words, speakers, utterances and JSON-able output.
"""
import json, os, sys
HERE = os.path.dirname(os.path.abspath(__file__)); ROOT = os.path.dirname(HERE); sys.path.insert(0, ROOT)
from arctag.asr import DeepgramAdapter, SpeechmaticsAdapter, summarize # noqa: E402
FIX = os.path.join(HERE, "fixtures")
U1 = ["Hi,", "this", "is", "Dana", "from", "Acme", "Telecom."]
U2 = ["How", "can", "I", "help", "you", "today?"]
U3 = ["Yeah,", "my", "name", "is", "John", "Patel."]
U4 = ["And", "my", "number", "is", "555-213-4489."]
U5 = ["Thanks,", "John.", "I'll", "send", "a", "text", "to", "that", "number."]
ALL = U1 + U2 + U3 + U4 + U5
def P(words, final, k):
return ("push_words", {"words": words, "final": final, "revise_from": k})
E = ("endpoint", {})
EXPECT = {
"deepgram": [P(["hi", "this", "is"], False, 0), P(["hi", "this", "is", "dana", "from"], False, 0), P(U1, True, 0),
P(["how", "can", "i"], False, 7), P(U2, True, 7), E,
P(["yeah", "my", "name"], False, 13), P(["yeah", "my", "name", "is", "john"], False, 13), P(U3, True, 13),
P(["and", "my", "number", "is", "five", "five", "five"], False, 19),
P(U4, True, 19), E, # UtteranceEnd arrived with interim words pending: endpoint after the final
P(["thanks", "john"], False, 24), P(U5, True, 24), E],
"speechmatics": [P(["Hi", "this"], False, 0), P(["Hi,", "this", "is", "Dana"], False, 0), P(U1, True, 0),
P(["How", "can", "I"], False, 7), P(U2, True, 7), E,
P(["Yeah", "my", "name"], False, 13), P(U3, True, 13),
P(["And", "my", "number", "is", "555"], False, 19),
P(U4, True, 19), E, # EndOfUtterance arrived with partial words pending
P(["Thanks", "John"], False, 24), P(U5, True, 24), E],
}
ADAPTERS = {"deepgram": (DeepgramAdapter, "deepgram_live.jsonl"), "speechmatics": (SpeechmaticsAdapter, "speechmatics_rt.jsonl")}
SPK = {"deepgram": ("S0", "S1"), "speechmatics": ("S1", "S2")}
def messages(name):
return [json.loads(l) for l in open(os.path.join(FIX, ADAPTERS[name][1])) if l.strip()]
class FakeStream:
"""records calls; keeps the word list like the real stream does"""
def __init__(self):
self.words = []; self.n_endpoints = 0; self.spk = []
def push_words(self, words, final=True, revise_from=None, endpoint=False):
k = len(self.words) if revise_from is None else revise_from
assert 0 <= k <= len(self.words)
self.words = self.words[:k] + [dict(w, final=final) for w in words]
return {"n_words": len(self.words), "spans": [], "events": [], "utterances": [], "running": {}}
def endpoint(self):
assert all(w["final"] for w in self.words), "endpoint with non-final words"
self.n_endpoints += 1
return {"n_words": len(self.words), "spans": [], "events": [], "utterances": [{}], "running": {}}
def redacted_text(self):
return " ".join(w["w"] for w in self.words)
def check_sequence(name):
cls, _ = ADAPTERS[name]
fs = FakeStream(); ad = cls(fs)
for m in messages(name):
ad(m)
assert ad.calls == EXPECT[name], json.dumps(ad.calls, indent=0)
assert [w["w"] for w in fs.words] == ALL and all(w["final"] for w in fs.words)
spk = [w.get("spk") for w in fs.words]
a, b = SPK[name]
assert spk == [a] * 13 + [b] * 11 + [a] * 9, spk
assert fs.n_endpoints == 3
return len(ad.calls)
def check_end_to_end(name, tg):
cls, _ = ADAPTERS[name]
st = tg.stream(domain="talk"); ad = cls(st); lines = []
for m in messages(name):
for up in ad(m):
lines.append(json.dumps(summarize(up, st))) # must be JSON-serialisable
res = st.finalize()
assert res["words"] == ALL, res["words"]
assert [(u["start"], u["end"]) for u in res["utterances"]] == [(0, 13), (13, 24), (24, 33)], res["utterances"]
assert [w.get("spk") for w in st.s.words][:2] == [SPK[name][0]] * 2
return {"updates": len(lines), "redacted_text": res["redacted_text"],
"spans": [(s["tag"], s["text"]) for s in res["spans"]],
"acts": [u["act"] for u in res["utterances"]], "roles": [u["role"] for u in res["utterances"]]}
def test_deepgram_sequence():
check_sequence("deepgram")
def test_speechmatics_sequence():
check_sequence("speechmatics")
def test_end_to_end():
from arctag import ArcTagger
tg = ArcTagger.from_pretrained(ROOT)
for name in ADAPTERS:
check_end_to_end(name, tg)
if __name__ == "__main__":
for name in ADAPTERS:
print(f"{name}: call sequence OK ({check_sequence(name)} calls)")
import torch
torch.set_num_threads(1)
from arctag import ArcTagger
tg = ArcTagger.from_pretrained(ROOT)
for name in ADAPTERS:
print(f"{name}: end to end OK", json.dumps(check_end_to_end(name, tg)))