translator / opus_mt.py
Rodion111's picture
update
8d34056 verified
Raw History Blame Contribute Delete
6.84 kB
"""Local Helsinki-NLP opus-mt he<->ru fallback translator (CTranslate2 int8 + sentencepiece, no torch).
Models dir: env OPUS_DIR (default ./models), subdirs opus-he-ru, opus-ru-he (ct2 int8 + source.spm/target.spm)."""
import os, re, difflib, threading, time, pathlib, logging
log = logging.getLogger("tr")
DIR = pathlib.Path(os.environ.get("OPUS_DIR", pathlib.Path(__file__).with_name("models")))
_M, _lock = {}, threading.Lock()
def available():
return all((DIR / f"opus-{p}" / "model.bin").exists() for p in ("he-ru", "ru-he"))
def _load(pair):
with _lock:
if pair not in _M:
import ctranslate2, sentencepiece as spm
d = DIR / f"opus-{pair}"
tr = ctranslate2.Translator(str(d), device="cpu", compute_type="int8",
inter_threads=1, intra_threads=int(os.environ.get("OPUS_THREADS", "2")))
_M[pair] = (tr, spm.SentencePieceProcessor(model_file=str(d / "source.spm")),
spm.SentencePieceProcessor(model_file=str(d / "target.spm")))
return _M[pair]
def preload():
for p in ("he-ru", "ru-he"): _load(p)
# ---------------- glossary (tested on mt_bench/glossary: 27/27 term hits, 1 of 22 sentences needs Gemini) ----------------
HE_PREF = "ובלמהשכ"
def is_he(t): return bool(re.search(r"[\u0590-\u05FF]", t))
def parse(gl):
g=[]
for line in (gl or "").splitlines():
if "=" in line:
a,b=(x.strip() for x in line.split("=",1))
if a and b: g.append((a,b))
return g
def ru_stem(w):
w=w.lower(); return re.sub(r"[аеиоуыэюяйь]+$","",w) if len(w)>3 else w
def src_pattern(term):
"""Regex for a glossary term in source text: Hebrew allows 0-2 prefix letters; Russian allows inflection endings."""
words=re.split(r"([\s\-]+)",term)
if is_he(term):
body="".join(re.escape(w) if i%2==0 else r"[\s\-]+" for i,w in enumerate(words))
return re.compile(rf"(?<![\w\u0590-\u05FF])([{HE_PREF}]{{0,2}})({body})(?![\w\u0590-\u05FF])")
parts=[]
for i,w in enumerate(words):
if i%2: parts.append(r"[\s\-]+")
else:
st=ru_stem(w); parts.append(re.escape(st)+r"[а-яё]{0,3}")
return re.compile(r"(?<![\w])()("+"".join(parts)+r")(?![\w])", re.I)
def find_terms(text, gl):
"""-> list of (src_term, tgt_term) whose source side occurs in text, longest first, both directions."""
pairs=parse(gl); pairs=pairs+[(b,a) for a,b in pairs]
src_he=is_he(text); out=[]
for a,b in sorted(pairs,key=lambda x:-len(x[0])):
if is_he(a)!=src_he: continue
if src_pattern(a).search(text): out.append((a,b))
return out
# --- skeletons for fuzzy matching
HE_MAP=str.maketrans({"ק":"כ","ט":"ת","ס":"ש","ך":"כ","ם":"מ","ן":"נ","ף":"פ","ץ":"צ","ב":"ב","'":"","־":"","-":""," ":""})
RU_MAP=str.maketrans({"х":"к","ё":"е","й":"и","ъ":"","ь":"","-":""," ":""})
def skel(t):
if is_he(t):
t=t.translate(HE_MAP); return re.sub("[ויאהע]","",t) or t
t=t.lower().translate(RU_MAP); t=re.sub(r"(.)\1",r"\1",t) # кк->к
return re.sub("[аеиоуыэюя]","",t) or t
def sim(a,b): return difflib.SequenceMatcher(None,skel(a),skel(b)).ratio()
def fuzzy_sub(out, tgt, thr=0.6):
"""Replace the output span most similar to glossary target `tgt` by `tgt` (keeps Hebrew prefix / Russian ending)."""
if tgt.lower() in out.lower(): return out, True
toks=list(re.finditer(r"[\w\u0590-\u05FF'׳]+(?:-[\w\u0590-\u05FF'׳]+)*", out))
if len(skel(tgt))<=3: thr=max(thr,0.75)
n=len(tgt.split()); best=(0,None)
for i in range(len(toks)):
for k in range(max(1,n-1),n+2):
if i+k>len(toks): break
s,e=toks[i].start(),toks[i+k-1].end(); span=out[s:e]
core=span; pre=""; suf=""
if is_he(tgt):
m=re.match(f"^([{HE_PREF}]{{0,2}})(.+)$",span)
cands=[(m.group(1)[:j],span[j:]) for j in range(len(m.group(1))+1)]
else:
cands=[("",span)]
for p,c in cands:
if not is_he(tgt):
# allow inflected ending on the last word
for cut in range(0,4):
cc=c[:len(c)-cut] if cut else c
if len(cc)<2: break
r=sim(cc,tgt)
if r>best[0]: best=(r,(s,e,p,c[len(c)-cut:] if cut else ""))
else:
r=sim(c,tgt)
if r>best[0]: best=(r,(s,e,p,""))
if best[1] and best[0]>=thr:
s,e,p,suf=best[1]
if not is_he(tgt) and suf and not re.fullmatch(r"[аеиоуыюяйьём]{1,3}",suf): suf=""
return out[:s]+p+tgt+suf+out[e:], True
return out, False
def _raw(text, target):
pair = "ru-he" if target == "he" else "he-ru"
tr, ss, ts = _load(pair)
out = []
for sent in [s for s in re.split(r"(?<=[.!?…])\s+", text.strip()) if s] or [text]:
toks = ss.encode(sent, out_type=str) + ["</s>"]
r = tr.translate_batch([toks], beam_size=4, max_decoding_length=200, repetition_penalty=1.1)
out.append(ts.decode(r[0].hypotheses[0]))
return re.sub(r"(\d{1,2}): (\d{2})\b", r"\1:\2", " ".join(out)) # '15: 30' -> '15:30'
def term_ok(out, tgt):
if is_he(tgt): return tgt in out
o = out.lower(); return all(ru_stem(w) in o for w in re.split(r"[\s\-]+", tgt.lower()))
PH_BAD = re.compile(r"ZQX|קיו|אקס|ЗКХ|зкх", re.I)
def translate_ex(text, target, glossary=""):
"""-> (translation, missing glossary targets, method). Strategy per glossary hit in the source:
1) plain translation + fuzzy post-substitution of the model's transliteration (most fluent);
2) source term -> placeholder 'ZQX<n>' (opus-mt copies it), then put the target term back.
The caller may fall back to an LLM when terms are still missing."""
terms = find_terms(text, glossary)
out = _raw(text, target)
if not terms: return out, [], "plain"
best = None
for a, b in terms: out, _ = fuzzy_sub(out, b)
miss = [b for a, b in terms if not term_ok(out, b)]
if not miss: return out, [], "fuzzy"
best = (out, miss, "fuzzy")
src, ph = text, {}
for i, (a, b) in enumerate(terms):
key = f"ZQX{i + 1}"; src = src_pattern(a).sub(lambda m: m.group(1) + key, src); ph[key] = b
out2 = _raw(src, target)
for k, v in ph.items(): out2 = out2.replace(k, v)
miss2 = [b for a, b in terms if not term_ok(out2, b)]
if not PH_BAD.search(out2) and len(miss2) < len(miss): best = (out2, miss2, "placeholder")
return best
def translate(text, target, glossary=""):
"""Translate Hebrew->Russian (target='ru') or Russian->Hebrew (target='he'). Blocking; call in a thread."""
return translate_ex(text, target, glossary)[0]