"""Dependency-free English morphological segmentation. Splits a word into ``(prefix, root, suffix)`` using curated affix lists with longest-match and minimum-root guards. Deliberately simple and predictable: no dictionary, no learned model, no external deps. Imperfect (it won't undo consonant doubling like ``running -> run``) — documented as future work — but it handles the common derivational/inflectional cases the tokenizer relies on. """ from __future__ import annotations # Conservative, high-precision derivational prefixes (min root kept >= MIN_ROOT). PREFIXES: tuple[str, ...] = ( "anti", "auto", "circum", "contra", "counter", "dis", "extra", "fore", "hyper", "hypo", "in", "im", "il", "ir", "inter", "intra", "mal", "mid", "mis", "multi", "non", "out", "over", "poly", "post", "pre", "proto", "pseudo", "re", "retro", "semi", "sub", "super", "supra", "sur", "trans", "tri", "ultra", "un", "under", ) # Inflectional + common derivational suffixes, tried longest-first. SUFFIXES: tuple[str, ...] = ( # 5+ chars "ation", "ation", "ically", "iosity", "itious", "aceous", "acious", "ality", "ative", "ator", "fully", "ially", "ables", "ibles", "iness", # 4 chars "able", "ably", "ance", "ence", "ency", "hood", "ible", "ical", "iest", "isms", "ists", "ment", "ness", "ship", "tion", "wise", # 3 chars "acy", "age", "al", "ant", "ate", "dom", "ees", "ers", "est", "ful", "ial", "ies", "ily", "ing", "ion", "ish", "ism", "ist", "ity", "ive", "ize", "ity", "less", "ous", "ity", # 2 chars "al", "ed", "en", "er", "es", "ic", "ly", "or", "ty", # 1 char (high-precision only) "s", ) # Keep roots at least this long after affix removal (avoids eating short words). MIN_ROOT = 3 _SUFFIXES_BY_LEN = tuple(sorted({s for s in SUFFIXES if s}, key=len, reverse=True)) def segment(word: str) -> tuple[str, str, str]: """Return ``(prefix, root, suffix)`` for ``word``; affixes are "" when absent.""" w = (word or "").strip().lower() if not w.isalpha(): return ("", w, "") prefix = "" for p in PREFIXES: if w.startswith(p) and len(w) - len(p) >= MIN_ROOT: prefix = p break root = w[len(prefix):] suffix = "" for s in _SUFFIXES_BY_LEN: if root.endswith(s) and len(root) - len(s) >= MIN_ROOT: suffix = s root = root[: len(root) - len(s)] break return (prefix, root, suffix)