"""Local Helsinki-NLP opus-mt he<->ru fallback translator (CTranslate2 int8 + sentencepiece, no torch). Models dir: env OPUS_DIR (default ./models), subdirs opus-he-ru, opus-ru-he (ct2 int8 + source.spm/target.spm).""" import os, re, difflib, threading, time, pathlib, logging log = logging.getLogger("tr") DIR = pathlib.Path(os.environ.get("OPUS_DIR", pathlib.Path(__file__).with_name("models"))) _M, _lock = {}, threading.Lock() def available(): return all((DIR / f"opus-{p}" / "model.bin").exists() for p in ("he-ru", "ru-he")) def _load(pair): with _lock: if pair not in _M: import ctranslate2, sentencepiece as spm d = DIR / f"opus-{pair}" tr = ctranslate2.Translator(str(d), device="cpu", compute_type="int8", inter_threads=1, intra_threads=int(os.environ.get("OPUS_THREADS", "2"))) _M[pair] = (tr, spm.SentencePieceProcessor(model_file=str(d / "source.spm")), spm.SentencePieceProcessor(model_file=str(d / "target.spm"))) return _M[pair] def preload(): for p in ("he-ru", "ru-he"): _load(p) # ---------------- glossary (tested on mt_bench/glossary: 27/27 term hits, 1 of 22 sentences needs Gemini) ---------------- HE_PREF = "ובלמהשכ" def is_he(t): return bool(re.search(r"[\u0590-\u05FF]", t)) def parse(gl): g=[] for line in (gl or "").splitlines(): if "=" in line: a,b=(x.strip() for x in line.split("=",1)) if a and b: g.append((a,b)) return g def ru_stem(w): w=w.lower(); return re.sub(r"[аеиоуыэюяйь]+$","",w) if len(w)>3 else w def src_pattern(term): """Regex for a glossary term in source text: Hebrew allows 0-2 prefix letters; Russian allows inflection endings.""" words=re.split(r"([\s\-]+)",term) if is_he(term): body="".join(re.escape(w) if i%2==0 else r"[\s\-]+" for i,w in enumerate(words)) return re.compile(rf"(? list of (src_term, tgt_term) whose source side occurs in text, longest first, both directions.""" pairs=parse(gl); pairs=pairs+[(b,a) for a,b in pairs] src_he=is_he(text); out=[] for a,b in sorted(pairs,key=lambda x:-len(x[0])): if is_he(a)!=src_he: continue if src_pattern(a).search(text): out.append((a,b)) return out # --- skeletons for fuzzy matching HE_MAP=str.maketrans({"ק":"כ","ט":"ת","ס":"ש","ך":"כ","ם":"מ","ן":"נ","ף":"פ","ץ":"צ","ב":"ב","'":"","־":"","-":""," ":""}) RU_MAP=str.maketrans({"х":"к","ё":"е","й":"и","ъ":"","ь":"","-":""," ":""}) def skel(t): if is_he(t): t=t.translate(HE_MAP); return re.sub("[ויאהע]","",t) or t t=t.lower().translate(RU_MAP); t=re.sub(r"(.)\1",r"\1",t) # кк->к return re.sub("[аеиоуыэюя]","",t) or t def sim(a,b): return difflib.SequenceMatcher(None,skel(a),skel(b)).ratio() def fuzzy_sub(out, tgt, thr=0.6): """Replace the output span most similar to glossary target `tgt` by `tgt` (keeps Hebrew prefix / Russian ending).""" if tgt.lower() in out.lower(): return out, True toks=list(re.finditer(r"[\w\u0590-\u05FF'׳]+(?:-[\w\u0590-\u05FF'׳]+)*", out)) if len(skel(tgt))<=3: thr=max(thr,0.75) n=len(tgt.split()); best=(0,None) for i in range(len(toks)): for k in range(max(1,n-1),n+2): if i+k>len(toks): break s,e=toks[i].start(),toks[i+k-1].end(); span=out[s:e] core=span; pre=""; suf="" if is_he(tgt): m=re.match(f"^([{HE_PREF}]{{0,2}})(.+)$",span) cands=[(m.group(1)[:j],span[j:]) for j in range(len(m.group(1))+1)] else: cands=[("",span)] for p,c in cands: if not is_he(tgt): # allow inflected ending on the last word for cut in range(0,4): cc=c[:len(c)-cut] if cut else c if len(cc)<2: break r=sim(cc,tgt) if r>best[0]: best=(r,(s,e,p,c[len(c)-cut:] if cut else "")) else: r=sim(c,tgt) if r>best[0]: best=(r,(s,e,p,"")) if best[1] and best[0]>=thr: s,e,p,suf=best[1] if not is_he(tgt) and suf and not re.fullmatch(r"[аеиоуыюяйьём]{1,3}",suf): suf="" return out[:s]+p+tgt+suf+out[e:], True return out, False def _raw(text, target): pair = "ru-he" if target == "he" else "he-ru" tr, ss, ts = _load(pair) out = [] for sent in [s for s in re.split(r"(?<=[.!?…])\s+", text.strip()) if s] or [text]: toks = ss.encode(sent, out_type=str) + [""] r = tr.translate_batch([toks], beam_size=4, max_decoding_length=200, repetition_penalty=1.1) out.append(ts.decode(r[0].hypotheses[0])) return re.sub(r"(\d{1,2}): (\d{2})\b", r"\1:\2", " ".join(out)) # '15: 30' -> '15:30' def term_ok(out, tgt): if is_he(tgt): return tgt in out o = out.lower(); return all(ru_stem(w) in o for w in re.split(r"[\s\-]+", tgt.lower())) PH_BAD = re.compile(r"ZQX|קיו|אקס|ЗКХ|зкх", re.I) def translate_ex(text, target, glossary=""): """-> (translation, missing glossary targets, method). Strategy per glossary hit in the source: 1) plain translation + fuzzy post-substitution of the model's transliteration (most fluent); 2) source term -> placeholder 'ZQX' (opus-mt copies it), then put the target term back. The caller may fall back to an LLM when terms are still missing.""" terms = find_terms(text, glossary) out = _raw(text, target) if not terms: return out, [], "plain" best = None for a, b in terms: out, _ = fuzzy_sub(out, b) miss = [b for a, b in terms if not term_ok(out, b)] if not miss: return out, [], "fuzzy" best = (out, miss, "fuzzy") src, ph = text, {} for i, (a, b) in enumerate(terms): key = f"ZQX{i + 1}"; src = src_pattern(a).sub(lambda m: m.group(1) + key, src); ph[key] = b out2 = _raw(src, target) for k, v in ph.items(): out2 = out2.replace(k, v) miss2 = [b for a, b in terms if not term_ok(out2, b)] if not PH_BAD.search(out2) and len(miss2) < len(miss): best = (out2, miss2, "placeholder") return best def translate(text, target, glossary=""): """Translate Hebrew->Russian (target='ru') or Russian->Hebrew (target='he'). Blocking; call in a thread.""" return translate_ex(text, target, glossary)[0]