update from github upstream (8197d38919dee305eebf595d7224cf7a3da5348a)

README.md

@@ -1,5 +1,5 @@
 ---
-title: Romansh Lemmatizer Demo
+title: RUMLEM: Romansh Lemmatizer Demo
 emoji: 💻
 colorFrom: purple
 colorTo: yellow
@@ -12,18 +12,18 @@ pinned: false
 license: mit
 ---
 
-# Basic Lemmatizer for Romansh Varieties <span style="color:gray">(Beta)</span>: Demo
+# Dictionary-Based Lemmatizer for Romansh Varieties <span style="color:gray">(Beta)</span>: Demo
 
-This demo visualises the functionalities of the package "romansh_lemmatizer", available at:
+This demo visualises the functionalities of the package "rumlem", available at:
 
-https://github.com/ZurichNLP/romansh_lemmatizer
+https://github.com/ZurichNLP/rumlem
 
 The underlying Python package presents a basic dictionary-based lemmatizer for the Romansh language.
-Provided a Romansh text, the lemmatizer splits it into words and looks up each word in the [Pledari Grond](https://pledarigrond.ch/) dictionaries for the five standard Romansh idioms: Sursilvan, Sutsilvan, Surmiran, Puter, and Vallader, as well as the dictionary for Rumantsch Grischun.
+Provided a Romansh text, the lemmatizer splits it into words and looks up each word in the [Pledari Grond](https://pledarigrond.ch/) dictionaries of the five primary Romansh idioms Sursilvan, Sutsilvan, Surmiran, Puter and Vallader, as well as in the dictionary of the standard variety Rumantsch Grischun.
 
 For example, if a Romansh text contains the word _lavuraiva_, the lemmatizer traces the word back to the Vallader and Puter dictionaries:
 
-<img src="https://huggingface.co/spaces/ZurichNLP/romansh-lemmatizer/resolve/main/illustration.png"
+<img src="https://huggingface.co/spaces/ZurichNLP/rumlem/resolve/main/illustration.png"
      alt="illustration" width="400"/>
 
 Typical use cases for the lemmatizer include:

lemmatizer/__init__.py

@@ -1 +1,3 @@
-from lemmatizer.lemmatizer import Lemmatizer
+from .lemmatizer import Lemmatizer, Doc, Idiom, Lemma, MorphAnalysis
+
+__all__ = ["Lemmatizer", "Doc", "Idiom", "Lemma", "MorphAnalysis"]

lemmatizer/analyzer.py

@@ -2,9 +2,8 @@ import json
 from pathlib import Path
 import pickle
 import sys
-
-from jiwer import cer
-
+from rapidfuzz.distance import Levenshtein
+from collections import defaultdict
 import lemmatizer.edittree as edittree
 
 BASE_DIR = Path(__file__).parent
@@ -33,20 +32,23 @@ class Analyzer:
         with open(json_path, "r", encoding="utf-8") as f:
             self.dict = json.load(f)
 
-        lem = []
-        for v in self.dict.values():
-            lem += v["lemma"]
+        self.lemma = {lemma for v in self.dict.values() for lemma in v["lemma"]}
 
-        self.lemma = lem
-        if self.learned_et:
-            self.edit_trees = []
+        self.edit_trees = []
+        self.et_suffix_index = defaultdict(list)
 
+        if self.learned_et:
+            sys.modules["edittree"] = edittree
             for pos in "noun", "adj", "verb":
                 et_path = BASE_DIR / "edit_trees" / f"{self.idiom}" / f"{pos}" / "et.txt"
-                sys.modules["edittree"] = edittree
                 with open(et_path, "rb") as f:
                     self.edit_trees += pickle.load(f)
 
+            for et_pack in self.edit_trees:
+                suffix = self._get_expected_suffix(et_pack)
+                if suffix is not None:
+                    self.et_suffix_index[suffix].append(et_pack)
+
         self.in_voc = in_voc
 
         other_de_path = BASE_DIR / "other_de" / f"{self.idiom}.json"
@@ -54,70 +56,82 @@ class Analyzer:
         with open(other_de_path, "r", encoding="utf-8") as f:
             self.other_de = json.load(f)
 
-    def get_lemma(self, tok: str):
-        """Obtain lemma through table look up; backs off
-        to unsupervised edit tree rules if no lemma found
-        """
+        self.lemma_by_pos = defaultdict(set)
+        for v in self.dict.values():
+            for lemma, unimorph in zip(v["lemma"], v["unimorph"]):
+                if unimorph:
+                    pos = unimorph.split(";")[0]  # "N", "V", "ADJ", "V.PTCP"
+                    self.lemma_by_pos[pos].add(lemma)
+
+    def _get_expected_suffix(self, et_pack) -> str | None:
+        """Recurse down the right spine to find the expected suffix."""
+        node = et_pack["et"]
+        while node is not None:
+            if isinstance(node.val[0], str):  # leaf node
+                return node.val[0]
+            node = node.right
+        return None
+
+    def analyze(self, tok: str):
         tok = tok.lower().strip()
         entry = self.dict.get(tok)
-        entry_ls = []
-        if entry:
-            entry_ls.extend(entry["lemma"])
-        if tok in self.other_de: # Augment the results with other_de entries
-              amount = len(self.other_de[tok]) # we need to add the lemma as many times as there are de translations for correct zipping later
-              tok_ls = [tok] * amount
-              entry_ls.extend(tok_ls)
-        if entry_ls:
-            return entry_ls
-        
-        # Check if there's a lemma from the edit trees
-        if self.learned_et:
-            et_out = self._et_lemma(tok)
-            if et_out:
-                return [et_out]
 
-        # Assume the token is a lemma
-        return [tok] if tok in self.in_voc else [None]
+        lemmas, de_list, unimorph_list = [], [], []
 
-    def _et_lemma(self, tok: str):
+        if entry:
+            lemmas.extend(entry["lemma"])
+            de_list.extend(entry["DStichwort"])
+            unimorph_list.extend(entry["unimorph"])
+
+        if tok in self.other_de:
+            amount = len(self.other_de[tok])
+            lemmas.extend([tok] * amount)
+            de_list.extend(self.other_de[tok])
+            unimorph_list.extend([None] * amount)
+
+        if lemmas:
+            return lemmas, de_list, unimorph_list
+
+        if tok in self.in_voc:
+            return [tok], [None], [None]
+        return [None], [None], [None]
+        
+    def _et_analyze(self, tok: str):
         candidates = []
 
-        for et_pack in self.edit_trees:
-            et = et_pack["et"]
-            out = et.apply(tok)
-
-            if out != -1:
-                candidates.append(out)
-
-        strong = [c for c in candidates if c in self.lemma]
+        for suffix, packs in self.et_suffix_index.items():
+            if tok.endswith(suffix):
+                for et_pack in packs:
+                    out = et_pack["et"].apply(tok)
+                    if out != -1:
+                        candidates.append((out, et_pack.get("majority_tag")))
+
+        strong_tagged = []
+        strong_untagged = []
+
+        for c, tag in candidates:
+            if c not in self.lemma:
+                continue
+            if tag:
+                pos = tag.split(";")[0]
+                pos_lemmas = self.lemma_by_pos.get(pos, self.lemma)
+                if c in pos_lemmas:
+                    strong_tagged.append((c, tag))
+            else:
+                if len(c) <= len(tok):  # lemma can't be longer than inflected form
+                    strong_untagged.append((c, tag))
+
+        # Prefer tagged candidates; only use untagged if nothing else found
+        strong = strong_tagged if strong_tagged else strong_untagged
+
+        if not strong:
+            return None, None
 
         if len(strong) > 1:
-            # Choose the candidate with the lowest edit distance to the tok:
-            dist = {}
-            for c in strong:
-                dist[c] = cer(tok, c)
-            out = min(dist, key=dist.get)
-            return out if out in self.in_voc else None
-
-        return strong[0] if strong and strong[0] in self.in_voc else None
-
-    def get_unimorph(self, tok: str):
-        """Obtain Unimorph annotation for N, V, and ADJ
-        in the Pledari Grond Dict"""
-        tok = tok.lower().strip()
-        entry = self.dict.get(tok)
-        if entry:
-            return entry["unimorph"]
-        return [None]
-
-    def get_de(self, tok: str):
-        """Obtain the German word corresponding to Romansh terms in the Pledari Grond Dict"""
-        tok = tok.lower().strip()
-        entry = self.dict.get(tok)
-        entry_ls = []
-        if entry:
-            entry_ls.extend(entry["DStichwort"])
-        if tok in self.other_de: # Augment the results with other_de entries
-           entry_ls.extend(self.other_de[tok])
-        return entry_ls if entry_ls else [None]
+            dist = {c: Levenshtein.normalized_distance(tok, c) for c, _ in strong}
+            best = min(dist, key=dist.get)
+            match = next((c, tag) for c, tag in strong if c == best)
+            return match if match[0] in self.in_voc else (None, None)
 
+        c, tag = strong[0]
+        return (c, tag) if c in self.in_voc else (None, None)

lemmatizer/lemmatizer.py

@@ -157,28 +157,47 @@ class Lemmatizer:
 
     def __call__(self, text: str) -> Doc:
         # Tokenize the text
+        from itertools import zip_longest
         toks = self.tokenizer.tokenize(text)
-
         tok_obj = []
 
+        # First pass: dict only, all idioms
+        first_pass = {}  # tok -> {idiom: (lemmas, de, unimorph)}
         for t in toks:
-            full_lemma = defaultdict(list)
             t_lower = t.lower()
-            from itertools import zip_longest
-
+            first_pass[t_lower] = {}
             for idiom in Idiom:
-                a = self._analyzers[idiom]
-                lemma = a.get_lemma(t_lower)
-                de = a.get_de(t_lower)
-                unimorph = a.get_unimorph(t_lower)
-
-                # safer: align lists of possibly different length
-                for l, d, u in zip_longest(lemma, de, unimorph, fillvalue=None):
+                result = self._analyzers[idiom].analyze(t_lower)
+                if result[0] != [None]:
+                    first_pass[t_lower][idiom] = result
+
+        # Detect idiom
+        if self.idiom:
+            best_idiom = self.idiom
+        else:
+            scores = get_scores([t.lower() for t in toks], self.in_voc)
+            best_idiom = max(scores, key=scores.get)
+
+        # Second pass: ET for unresolved tokens, best idiom only
+        best_analyzer = self._analyzers[best_idiom]
+        for t_lower, idiom_results in first_pass.items():
+            result = idiom_results.get(best_idiom)
+            # Run ET if no result, or if the only result is the token itself (in_voc fallback)
+            if result is None or result == ([t_lower], [None], [None]):
+                et_lemma, et_tag = best_analyzer._et_analyze(t_lower)
+                if et_lemma:
+                    idiom_results[best_idiom] = ([et_lemma], [None], [et_tag])
+
+        # Build token objects
+        for t in toks:
+            t_lower = t.lower()
+            full_lemma = defaultdict(list)
+            for idiom, (lemmas, de_list, unimorph_list) in first_pass[t_lower].items():
+                for l, d, u in zip_longest(lemmas, de_list, unimorph_list, fillvalue=None):
                     if l:
                         lem = Lemma(idiom, l, d if d else "null")
                         analysis = MorphAnalysis(get_features(u))
                         full_lemma[lem].append(analysis)
-
             tok_obj.append(Token(t, full_lemma, self.idiom))
 
-        return Doc(text, tok_obj, self.in_voc, self.idiom,)
+        return Doc(text, tok_obj, self.in_voc, self.idiom)