| |
| |
| """ |
| Reproduce the turkish-stt WER/CER benchmark on a shared eval set. |
| |
| pip install sherpa-onnx soundfile librosa jiwer num2words huggingface_hub |
| python run_eval.py --data-dir tts-synthetic # weights auto-download from HF |
| # or with local weights: |
| python run_eval.py --data-dir tts-synthetic --model-dir /path/to/weights |
| |
| Uses the published weights (encoder.int8.onnx + decoder.onnx + joiner.int8.onnx), applies one |
| matched Turkish normalization, computes WER/CER with jiwer, and writes raw hypotheses to |
| hypotheses.tsv for independent verification. Tested with sherpa-onnx 1.13.4. |
| """ |
| import argparse, csv, os, re |
| import numpy as np, soundfile as sf, sherpa_onnx, jiwer |
| from num2words import num2words |
|
|
| REPO = "mihuai/turkish-stt" |
|
|
| def files(model_dir): |
| if model_dir: |
| j = lambda f: os.path.join(model_dir, f) |
| return j("tokens.txt"), j("encoder.int8.onnx"), j("decoder.onnx"), j("joiner.int8.onnx") |
| from huggingface_hub import hf_hub_download |
| d = lambda f: hf_hub_download(REPO, f) |
| return d("tokens.txt"), d("encoder.int8.onnx"), d("decoder.onnx"), d("joiner.int8.onnx") |
|
|
| def norm(t): |
| if not t: return "" |
| for a,b in [("İ","i"),("I","ı"),("Ş","ş"),("Ğ","ğ"),("Ü","ü"),("Ö","ö"),("Ç","ç")]: t=t.replace(a,b) |
| t=t.lower(); t=re.sub(r"(\d)\s+(\d)",r"\1\2",t) |
| t=re.sub(r"\d+",lambda m:num2words(int(m.group()),lang="tr"),t) |
| t=re.sub(r"[^a-zâîûçğıöşü\s]"," ",t); return re.sub(r"\s+"," ",t).strip() |
|
|
| def main(): |
| ap=argparse.ArgumentParser() |
| ap.add_argument("--data-dir", default="tts-synthetic") |
| ap.add_argument("--model-dir", default=None) |
| a=ap.parse_args() |
| tok,enc,dec,joi=files(a.model_dir) |
| rec=sherpa_onnx.OnlineRecognizer.from_transducer(tokens=tok,encoder=enc,decoder=dec,joiner=joi, |
| num_threads=2,decoding_method="modified_beam_search",provider="cpu") |
| def stt(p): |
| x,sr=sf.read(p,dtype="float32") |
| if x.ndim>1: x=x.mean(axis=1) |
| if sr!=16000: |
| import librosa; x=librosa.resample(x,orig_sr=sr,target_sr=16000) |
| s=rec.create_stream(); s.accept_waveform(16000,x); s.accept_waveform(16000,np.zeros(8000,dtype=np.float32)); s.input_finished() |
| while rec.is_ready(s): rec.decode_stream(s) |
| r=rec.get_result(s); return r if isinstance(r,str) else r.text |
| H,R,rows=[],[],[] |
| with open(os.path.join(a.data_dir,"manifest.tsv"),encoding="utf-8") as f: |
| for row in csv.DictReader(f,delimiter="\t"): |
| hyp=stt(os.path.join(a.data_dir,row["audio"])) |
| H.append(norm(hyp)); R.append(norm(row["reference"])) |
| rows.append((row["audio"],row["reference"],hyp)) |
| with open("hypotheses.tsv","w",encoding="utf-8") as f: |
| f.write("audio\treference\thypothesis\n") |
| for r in rows: f.write("\t".join(r)+"\n") |
| print("clips: %d | WER %.2f%% | CER %.2f%% (raw hypotheses -> hypotheses.tsv)"%( |
| len(R),jiwer.wer(R,H)*100,jiwer.cer(R,H)*100)) |
|
|
| if __name__=="__main__": main() |
|
|
