#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Reproduce the turkish-stt WER/CER benchmark on a shared eval set. pip install sherpa-onnx soundfile librosa jiwer num2words huggingface_hub python run_eval.py --data-dir tts-synthetic # weights auto-download from HF # or with local weights: python run_eval.py --data-dir tts-synthetic --model-dir /path/to/weights Uses the published weights (encoder.int8.onnx + decoder.onnx + joiner.int8.onnx), applies one matched Turkish normalization, computes WER/CER with jiwer, and writes raw hypotheses to hypotheses.tsv for independent verification. Tested with sherpa-onnx 1.13.4. """ import argparse, csv, os, re import numpy as np, soundfile as sf, sherpa_onnx, jiwer from num2words import num2words REPO = "mihuai/turkish-stt" def files(model_dir): if model_dir: j = lambda f: os.path.join(model_dir, f) return j("tokens.txt"), j("encoder.int8.onnx"), j("decoder.onnx"), j("joiner.int8.onnx") from huggingface_hub import hf_hub_download d = lambda f: hf_hub_download(REPO, f) return d("tokens.txt"), d("encoder.int8.onnx"), d("decoder.onnx"), d("joiner.int8.onnx") def norm(t): if not t: return "" for a,b in [("İ","i"),("I","ı"),("Ş","ş"),("Ğ","ğ"),("Ü","ü"),("Ö","ö"),("Ç","ç")]: t=t.replace(a,b) t=t.lower(); t=re.sub(r"(\d)\s+(\d)",r"\1\2",t) t=re.sub(r"\d+",lambda m:num2words(int(m.group()),lang="tr"),t) t=re.sub(r"[^a-zâîûçğıöşü\s]"," ",t); return re.sub(r"\s+"," ",t).strip() def main(): ap=argparse.ArgumentParser() ap.add_argument("--data-dir", default="tts-synthetic") ap.add_argument("--model-dir", default=None) a=ap.parse_args() tok,enc,dec,joi=files(a.model_dir) rec=sherpa_onnx.OnlineRecognizer.from_transducer(tokens=tok,encoder=enc,decoder=dec,joiner=joi, num_threads=2,decoding_method="modified_beam_search",provider="cpu") def stt(p): x,sr=sf.read(p,dtype="float32") if x.ndim>1: x=x.mean(axis=1) if sr!=16000: import librosa; x=librosa.resample(x,orig_sr=sr,target_sr=16000) s=rec.create_stream(); s.accept_waveform(16000,x); s.accept_waveform(16000,np.zeros(8000,dtype=np.float32)); s.input_finished() while rec.is_ready(s): rec.decode_stream(s) r=rec.get_result(s); return r if isinstance(r,str) else r.text H,R,rows=[],[],[] with open(os.path.join(a.data_dir,"manifest.tsv"),encoding="utf-8") as f: for row in csv.DictReader(f,delimiter="\t"): hyp=stt(os.path.join(a.data_dir,row["audio"])) H.append(norm(hyp)); R.append(norm(row["reference"])) rows.append((row["audio"],row["reference"],hyp)) with open("hypotheses.tsv","w",encoding="utf-8") as f: f.write("audio\treference\thypothesis\n") for r in rows: f.write("\t".join(r)+"\n") print("clips: %d | WER %.2f%% | CER %.2f%% (raw hypotheses -> hypotheses.tsv)"%( len(R),jiwer.wer(R,H)*100,jiwer.cer(R,H)*100)) if __name__=="__main__": main()