"""Word alignment from tiny parallel corpora, pure python. Two complementary signals: 1. Minimal-pair set difference: if two sentence pairs differ in exactly one token on each side, those tokens correspond. Exact and high-precision; these puzzles are constructed to contain such pairs. 2. Dice co-occurrence over the whole pair set: soft alignment for everything the minimal pairs don't cover. """ from __future__ import annotations from collections import Counter, defaultdict from itertools import combinations from typing import Dict, List, Tuple from .preprocess import Pair, strip_punct, tokenize def _toks(s: str) -> List[str]: return [strip_punct(t).casefold() for t in tokenize(s) if strip_punct(t)] def minimal_pair_links(pairs: List[Pair]) -> Counter: """Set-difference alignment: for every pair of examples whose source sides differ by exactly one token multiset element and likewise on target, link the differing tokens. Returns Counter[(src_tok, tgt_tok)] link strengths.""" links: Counter = Counter() toks = [(Counter(_toks(p.src)), Counter(_toks(p.tgt))) for p in pairs] for (s1, t1), (s2, t2) in combinations(toks, 2): ds1, ds2 = s1 - s2, s2 - s1 dt1, dt2 = t1 - t2, t2 - t1 # exactly one differing token on each side, in both examples if sum(ds1.values()) == 1 and sum(ds2.values()) == 1 \ and sum(dt1.values()) == 1 and sum(dt2.values()) == 1: a1, a2 = next(iter(ds1)), next(iter(ds2)) b1, b2 = next(iter(dt1)), next(iter(dt2)) links[(a1, b1)] += 2 # strong: attested by contrast links[(a2, b2)] += 2 # shared residue: tokens present in both examples also co-align weakly return links def dice_scores(pairs: List[Pair]) -> Dict[Tuple[str, str], float]: """Dice coefficient between source and target tokens across examples.""" src_count: Counter = Counter() tgt_count: Counter = Counter() co: Counter = Counter() for p in pairs: st, tt = set(_toks(p.src)), set(_toks(p.tgt)) for a in st: src_count[a] += 1 for b in tt: tgt_count[b] += 1 for a in st: for b in tt: co[(a, b)] += 1 return { (a, b): 2 * c / (src_count[a] + tgt_count[b]) for (a, b), c in co.items() } def _morph_backoff(pairs: List[Pair], scores) -> None: """Substring evidence from single-word glosses: if (moko = dog) is attested and token `namoko` co-occurs with `dog`, boost (namoko, dog) — inflected forms inherit their stem's translation. Applied in place.""" word_pairs = [ (_toks(p.src)[0], _toks(p.tgt)[0]) for p in pairs if len(_toks(p.src)) == 1 and len(_toks(p.tgt)) == 1 ] for (a, b) in list(scores.keys()): for w, x in word_pairs: if x == b and len(w) >= 3 and w in a and w != a: scores[(a, b)] += 2.0 # inflected src contains attested stem if w == a and len(x) >= 3 and x in b and x != b: scores[(a, b)] += 2.0 # inflected tgt contains attested stem def align(pairs: List[Pair]) -> Dict[str, List[Tuple[str, float]]]: """Combined alignment: src token -> ranked [(tgt token, score)]. Minimal-pair links dominate (score offset +1.0 per link unit); Dice fills in the rest; single-word glosses back off into inflected forms containing them. Scores are comparable only within one puzzle. """ links = minimal_pair_links(pairs) dice = dice_scores(pairs) scores: Dict[Tuple[str, str], float] = defaultdict(float) for k, v in dice.items(): scores[k] += v for k, v in links.items(): scores[k] += 1.0 * v _morph_backoff(pairs, scores) # competition ("explaining away"): a target token strongly claimed by # some other source is a worse candidate — demote it proportionally to # its best competing suitor. Breaks the pervasive co-occurrence ties of # 10-sentence corpora in favor of unclaimed targets. best_suitor: Dict[str, float] = defaultdict(float) second_suitor: Dict[str, float] = defaultdict(float) for (a, b), s in scores.items(): if s > best_suitor[b]: second_suitor[b] = best_suitor[b] best_suitor[b] = s elif s > second_suitor[b]: second_suitor[b] = s out: Dict[str, List[Tuple[str, float]]] = defaultdict(list) for (a, b), s in scores.items(): rival = second_suitor[b] if s >= best_suitor[b] else best_suitor[b] out[a].append((b, s - 0.3 * rival)) for a in out: out[a].sort(key=lambda x: -x[1]) return dict(out) def one_to_one(pairs: List[Pair]) -> Dict[str, str]: """Greedy 1:1 token alignment: highest-scoring links assigned first, each token used once. Sharper than independent argmax when several tokens tie on co-occurrence (small corpora make ties common).""" amap = align(pairs) edges = [(s, a, b) for a, cands in amap.items() for b, s in cands] edges.sort(key=lambda e: (-e[0], e[1], e[2])) taken_a, taken_b, out = set(), set(), {} for s, a, b in edges: if a not in taken_a and b not in taken_b: out[a] = b taken_a.add(a) taken_b.add(b) return out def best_translation(align_map: Dict[str, List[Tuple[str, float]]], tok: str) -> str: cands = align_map.get(tok.casefold(), []) return cands[0][0] if cands else ""