Spaces:
Running
Running
Download opus_mt.py from Rodion111/translator: direct link, hf CLI and curl.
- Browser
- Download file 6.84 kB
-
https://huggingface.co/spaces/Rodion111/translator/resolve/main/opus_mt.py
- Command line
-
hf download hf://spaces/Rodion111/translator/opus_mt.py
-
curl -L -o opus_mt.py https://huggingface.co/spaces/Rodion111/translator/resolve/main/opus_mt.py
6.84 kB
| """Local Helsinki-NLP opus-mt he<->ru fallback translator (CTranslate2 int8 + sentencepiece, no torch). | |
| Models dir: env OPUS_DIR (default ./models), subdirs opus-he-ru, opus-ru-he (ct2 int8 + source.spm/target.spm).""" | |
| import os, re, difflib, threading, time, pathlib, logging | |
| log = logging.getLogger("tr") | |
| DIR = pathlib.Path(os.environ.get("OPUS_DIR", pathlib.Path(__file__).with_name("models"))) | |
| _M, _lock = {}, threading.Lock() | |
| def available(): | |
| return all((DIR / f"opus-{p}" / "model.bin").exists() for p in ("he-ru", "ru-he")) | |
| def _load(pair): | |
| with _lock: | |
| if pair not in _M: | |
| import ctranslate2, sentencepiece as spm | |
| d = DIR / f"opus-{pair}" | |
| tr = ctranslate2.Translator(str(d), device="cpu", compute_type="int8", | |
| inter_threads=1, intra_threads=int(os.environ.get("OPUS_THREADS", "2"))) | |
| _M[pair] = (tr, spm.SentencePieceProcessor(model_file=str(d / "source.spm")), | |
| spm.SentencePieceProcessor(model_file=str(d / "target.spm"))) | |
| return _M[pair] | |
| def preload(): | |
| for p in ("he-ru", "ru-he"): _load(p) | |
| # ---------------- glossary (tested on mt_bench/glossary: 27/27 term hits, 1 of 22 sentences needs Gemini) ---------------- | |
| HE_PREF = "ובלמהשכ" | |
| def is_he(t): return bool(re.search(r"[\u0590-\u05FF]", t)) | |
| def parse(gl): | |
| g=[] | |
| for line in (gl or "").splitlines(): | |
| if "=" in line: | |
| a,b=(x.strip() for x in line.split("=",1)) | |
| if a and b: g.append((a,b)) | |
| return g | |
| def ru_stem(w): | |
| w=w.lower(); return re.sub(r"[аеиоуыэюяйь]+$","",w) if len(w)>3 else w | |
| def src_pattern(term): | |
| """Regex for a glossary term in source text: Hebrew allows 0-2 prefix letters; Russian allows inflection endings.""" | |
| words=re.split(r"([\s\-]+)",term) | |
| if is_he(term): | |
| body="".join(re.escape(w) if i%2==0 else r"[\s\-]+" for i,w in enumerate(words)) | |
| return re.compile(rf"(?<![\w\u0590-\u05FF])([{HE_PREF}]{{0,2}})({body})(?![\w\u0590-\u05FF])") | |
| parts=[] | |
| for i,w in enumerate(words): | |
| if i%2: parts.append(r"[\s\-]+") | |
| else: | |
| st=ru_stem(w); parts.append(re.escape(st)+r"[а-яё]{0,3}") | |
| return re.compile(r"(?<![\w])()("+"".join(parts)+r")(?![\w])", re.I) | |
| def find_terms(text, gl): | |
| """-> list of (src_term, tgt_term) whose source side occurs in text, longest first, both directions.""" | |
| pairs=parse(gl); pairs=pairs+[(b,a) for a,b in pairs] | |
| src_he=is_he(text); out=[] | |
| for a,b in sorted(pairs,key=lambda x:-len(x[0])): | |
| if is_he(a)!=src_he: continue | |
| if src_pattern(a).search(text): out.append((a,b)) | |
| return out | |
| # --- skeletons for fuzzy matching | |
| HE_MAP=str.maketrans({"ק":"כ","ט":"ת","ס":"ש","ך":"כ","ם":"מ","ן":"נ","ף":"פ","ץ":"צ","ב":"ב","'":"","־":"","-":""," ":""}) | |
| RU_MAP=str.maketrans({"х":"к","ё":"е","й":"и","ъ":"","ь":"","-":""," ":""}) | |
| def skel(t): | |
| if is_he(t): | |
| t=t.translate(HE_MAP); return re.sub("[ויאהע]","",t) or t | |
| t=t.lower().translate(RU_MAP); t=re.sub(r"(.)\1",r"\1",t) # кк->к | |
| return re.sub("[аеиоуыэюя]","",t) or t | |
| def sim(a,b): return difflib.SequenceMatcher(None,skel(a),skel(b)).ratio() | |
| def fuzzy_sub(out, tgt, thr=0.6): | |
| """Replace the output span most similar to glossary target `tgt` by `tgt` (keeps Hebrew prefix / Russian ending).""" | |
| if tgt.lower() in out.lower(): return out, True | |
| toks=list(re.finditer(r"[\w\u0590-\u05FF'׳]+(?:-[\w\u0590-\u05FF'׳]+)*", out)) | |
| if len(skel(tgt))<=3: thr=max(thr,0.75) | |
| n=len(tgt.split()); best=(0,None) | |
| for i in range(len(toks)): | |
| for k in range(max(1,n-1),n+2): | |
| if i+k>len(toks): break | |
| s,e=toks[i].start(),toks[i+k-1].end(); span=out[s:e] | |
| core=span; pre=""; suf="" | |
| if is_he(tgt): | |
| m=re.match(f"^([{HE_PREF}]{{0,2}})(.+)$",span) | |
| cands=[(m.group(1)[:j],span[j:]) for j in range(len(m.group(1))+1)] | |
| else: | |
| cands=[("",span)] | |
| for p,c in cands: | |
| if not is_he(tgt): | |
| # allow inflected ending on the last word | |
| for cut in range(0,4): | |
| cc=c[:len(c)-cut] if cut else c | |
| if len(cc)<2: break | |
| r=sim(cc,tgt) | |
| if r>best[0]: best=(r,(s,e,p,c[len(c)-cut:] if cut else "")) | |
| else: | |
| r=sim(c,tgt) | |
| if r>best[0]: best=(r,(s,e,p,"")) | |
| if best[1] and best[0]>=thr: | |
| s,e,p,suf=best[1] | |
| if not is_he(tgt) and suf and not re.fullmatch(r"[аеиоуыюяйьём]{1,3}",suf): suf="" | |
| return out[:s]+p+tgt+suf+out[e:], True | |
| return out, False | |
| def _raw(text, target): | |
| pair = "ru-he" if target == "he" else "he-ru" | |
| tr, ss, ts = _load(pair) | |
| out = [] | |
| for sent in [s for s in re.split(r"(?<=[.!?…])\s+", text.strip()) if s] or [text]: | |
| toks = ss.encode(sent, out_type=str) + ["</s>"] | |
| r = tr.translate_batch([toks], beam_size=4, max_decoding_length=200, repetition_penalty=1.1) | |
| out.append(ts.decode(r[0].hypotheses[0])) | |
| return re.sub(r"(\d{1,2}): (\d{2})\b", r"\1:\2", " ".join(out)) # '15: 30' -> '15:30' | |
| def term_ok(out, tgt): | |
| if is_he(tgt): return tgt in out | |
| o = out.lower(); return all(ru_stem(w) in o for w in re.split(r"[\s\-]+", tgt.lower())) | |
| PH_BAD = re.compile(r"ZQX|קיו|אקס|ЗКХ|зкх", re.I) | |
| def translate_ex(text, target, glossary=""): | |
| """-> (translation, missing glossary targets, method). Strategy per glossary hit in the source: | |
| 1) plain translation + fuzzy post-substitution of the model's transliteration (most fluent); | |
| 2) source term -> placeholder 'ZQX<n>' (opus-mt copies it), then put the target term back. | |
| The caller may fall back to an LLM when terms are still missing.""" | |
| terms = find_terms(text, glossary) | |
| out = _raw(text, target) | |
| if not terms: return out, [], "plain" | |
| best = None | |
| for a, b in terms: out, _ = fuzzy_sub(out, b) | |
| miss = [b for a, b in terms if not term_ok(out, b)] | |
| if not miss: return out, [], "fuzzy" | |
| best = (out, miss, "fuzzy") | |
| src, ph = text, {} | |
| for i, (a, b) in enumerate(terms): | |
| key = f"ZQX{i + 1}"; src = src_pattern(a).sub(lambda m: m.group(1) + key, src); ph[key] = b | |
| out2 = _raw(src, target) | |
| for k, v in ph.items(): out2 = out2.replace(k, v) | |
| miss2 = [b for a, b in terms if not term_ok(out2, b)] | |
| if not PH_BAD.search(out2) and len(miss2) < len(miss): best = (out2, miss2, "placeholder") | |
| return best | |
| def translate(text, target, glossary=""): | |
| """Translate Hebrew->Russian (target='ru') or Russian->Hebrew (target='he'). Blocking; call in a thread.""" | |
| return translate_ex(text, target, glossary)[0] | |