voxi-lite / tests /check_release.py
Joshuant's picture
Voxi Lite 0.4.0: American and British English voices (harper, ethan, amelia, oliver on Kokoro-82M), CPU comparison on the card, English-accents samples
f746679 verified
Raw History Blame Contribute Delete
6.24 kB
"""Voxi Lite release checks: edge cases (must not crash, must behave as documented) and CPU speed.
python tests/check_release.py [--threads 4] [--out report.json]
Speed: per line, time to the first streamed chunk (first sentence) and real-time factor (audio seconds / compute
seconds, > 1 = faster than real time), on this machine's CPU. Run it on the production server to get its numbers.
"""
import argparse
import json
import platform
import sys
import time
from pathlib import Path
import numpy as np
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from voxi import MAX_CHARS, SR, VoxiLite # noqa: E402
EDGE = [
("empty", "", "hi", "empty"),
("spaces", " \n\t ", "hi", "empty"),
("punct_only", "...!!! ।", "hi", "empty"),
("emoji", "आपका ऑर्डर आ गया है 😊🎉", "hi", "audio"),
("control_chars", "Your order\x00 is\x07 ready.", "en", "audio"),
("numbers_dense_hi", "₹1,250, 10:30 बजे, OTP 482913, फ़ोन 98765 43210, 2.5 लाख, 31/12/2027", "hi", "audio"),
("numbers_ta", "உங்கள் OTP 482913. அழைக்க 98765 43210.", "ta", "audio"),
("mixed_script_hi", "आपका order ID AB1234 है, please confirm करें।", "hi", "audio"),
("hinglish", "Aapka bill ₹1,250 hai, kal tak pay kar dijiye.", "hinglish", "audio"),
("tanglish", "Unga bill ₹1,250 irukku, 15th-kulla pay pannunga.", "tanglish", "audio"),
("url_email", "Visit www.example.com or write to help@example.com.", "en", "audio"),
("single_word", "हाँ", "hi", "audio"),
("question", "क्या मैं आपकी और कोई मदद कर सकती हूँ?", "hi", "audio"),
("long_paragraph", " ".join(["आपका ऑर्डर कल शाम तक पहुँच जाएगा।"] * 25), "hi", "audio"),
("too_long", "क" * (MAX_CHARS + 1), "hi", "ValueError"),
("bad_lang", "hello", "xx", "ValueError"),
("bad_voice", "hello", "en", "ValueError:voice"),
("other_lang_te", "మీ టికెట్ కన్ఫర్మ్ అయింది.", "te", "audio"),
("style_voice", "माफ़ कीजिए, आपका रिफ़ंड अभी प्रोसेस हो रहा है।", "hi", "audio:payal@sad-soft"),
("en_us_voice", "Your refund of $1,250 has been processed. Call us at 800 555 0199.", "en", "audio:harper"),
("en_gb_voice", "I'm sorry about the delay; your parcel will arrive on Thursday.", "en", "audio:oliver"),
("en_voice_wrong_lang", "आपका ऑर्डर आ गया है।", "hi", "ValueError:amelia"),
]
SPEED = [
("hi_short", "जी, बिल्कुल। मैं अभी चेक करती हूँ।", "hi"),
("hi_numbers", "आपका बकाया ₹1,250 है, जो 15 तारीख़ तक भरना है। क्या मैं पेमेंट लिंक भेज दूँ?", "hi"),
("ta_agent", "உங்கள் ஆர்டர் நாளை வந்துவிடும். வேறு ஏதாவது உதவி வேண்டுமா?", "ta"),
("en_agent", "Thank you for calling. Your refund has been processed and will reach you in three to five days.", "en"),
("hinglish", "Namaste, main Voxi bol rahi hoon. Aapka order kal tak deliver ho jayega.", "hinglish"),
]
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--threads", type=int, default=4); ap.add_argument("--out", default="")
a = ap.parse_args()
t0 = time.time(); tts = VoxiLite(threads=a.threads); load_s = time.time() - t0
tts.synthesize("नमस्ते।", "hi") # warm-up
rep = {"machine": f"{platform.processor() or platform.machine()} / {platform.system()}", "threads": a.threads,
"load_s": round(load_s, 1), "edge": [], "speed": []}
ok = True
for name, text, lang, want in EDGE:
want, _, v = want.partition(":")
voice = "nobody" if v == "voice" else (v or "payal")
try:
y = tts.synthesize(text, lang, voice)
got = "empty" if len(y) == 0 else "audio"
bad = got == "audio" and (not np.isfinite(y).all() or np.abs(y).max() > 1.5 or len(y) < 0.2 * SR)
res = {"case": name, "want": want, "got": got, "audio_s": round(len(y) / SR, 2), "pass": got == want and not bad}
except ValueError as e:
res = {"case": name, "want": want, "got": "ValueError", "msg": str(e)[:80], "pass": want.startswith("ValueError")}
except Exception as e: # noqa: BLE001
res = {"case": name, "want": want, "got": type(e).__name__, "msg": str(e)[:120], "pass": False}
ok &= res["pass"]; rep["edge"].append(res)
print(("PASS " if res["pass"] else "FAIL ") + json.dumps(res, ensure_ascii=False), flush=True)
for name, text, lang in SPEED:
for voice in ("payal", "arjun", "sanjana", "vishnu") + (("harper", "ethan", "amelia", "oliver") if lang == "en" else ()):
t0 = time.time(); first = None; n = 0
for chunk in tts.stream(text, lang, voice):
first = first if first is not None else time.time() - t0
n += len(chunk)
tot = time.time() - t0
r = {"line": name, "voice": voice, "first_chunk_s": round(first, 2), "total_s": round(tot, 2),
"audio_s": round(n / SR, 2), "rtf_x": round(n / SR / tot, 2)}
rep["speed"].append(r); print("SPEED", json.dumps(r), flush=True)
s = rep["speed"]
rep["summary"] = {"edge_pass": f"{sum(r['pass'] for r in rep['edge'])}/{len(rep['edge'])}",
"first_chunk_s_median": float(np.median([r["first_chunk_s"] for r in s])),
"first_chunk_s_max": max(r["first_chunk_s"] for r in s),
"rtf_x_median": float(np.median([r["rtf_x"] for r in s]))}
print("SUMMARY", json.dumps(rep["summary"]), flush=True)
if a.out:
Path(a.out).write_text(json.dumps(rep, indent=1, ensure_ascii=False), encoding="utf-8")
sys.exit(0 if ok else 1)
if __name__ == "__main__":
main()