tigmorph / web.py
beshiribrahim's picture
Upload 4 files
c8085d2 verified
Raw History Blame Contribute Delete
2.74 kB
"""Browser glue for tigmorph (runs in Pyodide). Every function returns JSON for the page."""
import json
import re
import unicodedata
from collections import Counter
from functools import lru_cache
import tigmorph
from tigmorph import Analyzer, Normalizer
from tigmorph.fidel import decompose
A = Analyzer()
_NORM = {}
NOMINAL_TAMS = {'PartAct', 'PartPass', 'Agent', 'Inf', 'Instr'}
PUNCT = re.compile(r'^[\W_።፣፤፥፦፧፨፡]+|[\W_።፣፤፥፦፧፨፡]+$')
VERSION = tigmorph.__version__
def _class(a):
if 'POS' in a.tags:
return a.tags['POS']
return 'V>N' if a.tags.get('TAM') in NOMINAL_TAMS else 'V'
def _features(a):
tags = dict(a.tags)
tags.pop('POS', None)
if tags.get('TAM') in NOMINAL_TAMS and 'Subj' in tags:
tags['Num'] = tags.pop('Subj')[1:]
if a.guess:
tags['Guess'] = 'Yes'
return [[k, str(v)] for k, v in tags.items()]
def _reading(a):
return {'root': a.root, 'cls': _class(a), 'feats': _features(a), 'raz': a.raz, 'gloss': a.gloss,
'score': round(a.weight, 2), 'guess': a.guess}
def analyze(text, n, guess, latin, compound):
text = unicodedata.normalize('NFC', text or '').strip()
if not text:
return json.dumps([])
g = True if guess else 'auto'
if compound:
return json.dumps([{'word': text, 'readings': [_reading(a) for a in A.analyze(text, guess=g)[:n]]}])
out = []
for t in re.split(r'[\s፡]+', text)[:200]:
t = PUNCT.sub('', t)
if not t:
continue
res = A.analyze_latin(t, guess=g) if latin else A.analyze(t, guess=g)
out.append({'word': t, 'readings': [_reading(a) for a in res[:n]]})
return json.dumps(out)
def root_to_latin(root):
root = unicodedata.normalize('NFC', root.strip())
if any('ሀ' <= c <= '᎟' or 'ⶀ' <= c <= '⷟' for c in root):
return '-'.join(decompose(c)[0] for c in root if c not in '- ')
if '-' in root:
return root.replace(' ', '')
return '-'.join(re.findall(r'.[̀-ͯ]*', root.replace(' ', '')))
@lru_cache(maxsize=20000)
def _gen(root, key):
try:
return tuple(A.generate(root, dict(key)))
except KeyError:
return ()
def generate(root, tags_json):
r = root_to_latin(root)
tags = {k: v for k, v in json.loads(tags_json).items() if v}
forms = _gen(r, tuple(sorted(tags.items())))
return json.dumps({'root': r, 'forms': [[raz, gz] for raz, gz in forms]})
def normalize(text, level):
if level not in _NORM:
_NORM[level] = Normalizer(level=level, analyzer=A)
log = Counter()
out = _NORM[level].normalize(text or '', log=log)
return json.dumps({'text': out, 'changes': [[a, b, c] for (a, b), c in log.most_common()]})