from __future__ import absolute_import, print_function, unicode_literals import codecs import copy import os.path import pkg_resources import yaml import regex as re import unicodecsv as csv from . import _panphon, xsampa def flip(s): return [(b, a) for (a, b) in s] def update_ft_set(seg, dia): seg = dict(flip(seg)) seg.update(dia) return flip(set(seg.items())) class PermissiveFeatureTable(_panphon.FeatureTable): """Encapsulate the segment <=> feature vector mapping implied by the files data/ipa_all.csv and diacritic_definitions.yml. Uses a more permissive algorithm for identifying base+diacritic combinations. To avoid a combinatorial explosion, it never generates all of the base-diacritic- modifier combinations, meaning it cannot easily make statements about the whole set of segments.""" def __init__(self, feature_set='spe+', feature_model='strict', ipa_bases=os.path.join('data', 'ipa_bases.csv'), dias=os.path.join('data', 'diacritic_definitions.yml'), ): """Construct a PermissiveFeatureTable object Args: feature_set (str): feature system (for API compatibility) feature_model (str): feature parsing model (for API compatibility) ipa_bases (str): path from panphon root to CSV file definining features of bases (unmodified consonants and vowels) dias (str): path from panphon root to YAML file containing rules for diacritics and modifiers """ dias = pkg_resources.resource_filename(__name__, dias) self.bases, self.names = self._read_ipa_bases(ipa_bases) self.prefix_dias, self.postfix_dias = self._read_dias(dias) self.pre_regex, self.post_regex, self.seg_regex = self._compile_seg_regexes(self.bases, self.prefix_dias, self.postfix_dias) self.xsampa = xsampa.XSampa() self.weights = self._read_weights() def _read_ipa_bases(self, fn): fn = pkg_resources.resource_filename(__name__, fn) with open(fn, 'rb') as f: reader = csv.reader(f, encoding='utf-8', delimiter=str(',')) names = next(reader)[1:] bases = {} for row in reader: seg, vals = row[0], row[1:] bases[seg] = (set(zip(vals, names))) return bases, names def _read_dias(self, fn): prefix, postfix = {}, {} with codecs.open(fn, 'r', 'utf-8') as f: defs = yaml.load(f.read(), Loader=yaml.FullLoader) for dia in defs['diacritics']: if dia['position'] == 'pre': prefix[dia['marker']] = dia['content'] else: postfix[dia['marker']] = dia['content'] return prefix, postfix def _compile_seg_regexes(self, bases, prefix, postfix): pre_jnd = '|'.join(prefix.keys()) post_jnd = '|'.join(postfix.keys()) bases_jnd = '|'.join(bases.keys()) pre_re = '({})'.format(pre_jnd) post_re = '({})'.format(post_jnd) seg_re = '(?P(?P
({})*)(?P{})(?P({})*))'.format(pre_jnd, bases_jnd, post_jnd)
        return re.compile(pre_re), re.compile(post_re), re.compile(seg_re)

    def _build_seg_regex(self):
        return self.seg_regex

    def _read_weights(self, filename=os.path.join('data', 'feature_weights.csv')):
        filename = pkg_resources.resource_filename(
            __name__, filename)
        with open(filename, 'rb') as f:
            reader = csv.reader(f, encoding='utf-8')
            next(reader)
            weights = [float(x) for x in next(reader)]
        return weights

    def fts(self, segment):
        """Return features corresponding to segment as list of (value,
        feature) tuples

        Args:
            segment (unicode): segment for which features are to be returned as
                               Unicode string

        Returns:
            list: None if `segment` cannot be parsed; otherwise, a list of the
                  features of `segment` as (value, feature) pairs
        """
        match = self.seg_regex.match(segment)
        if match:
            pre, base, post = match.group('pre'), match.group('base'), match.group('post')
            seg = copy.deepcopy(self.bases[base])
            for m in reversed(pre):
                seg = update_ft_set(seg, self.prefix_dias[m])
            for m in post:
                seg = update_ft_set(seg, self.postfix_dias[m])
            return set(seg)
        else:
            return None

    def fts_match(self, fts_mask, segment):
        """Evaluates whether a set of features 'match' a segment (are a subset
        of that segment's features)

        Args:
            fts_mask (list): list of (value, feature) tuples
            segment (unicode): IPA string corresponding to segment (consonant or
                               vowel)
        Returns:
            bool: None if `segment` cannot be parsed; True if the feature values
                  of `fts_mask` are a subset of those for `segment`
        """
        fts_seg = self.fts(segment)
        if fts_seg:
            fts_mask = set(fts_mask)
            return fts_mask <= fts_seg
        else:
            return None

    def longest_one_seg_prefix(self, word):
        """Return longest IPA Unicode prefix of `word`

        Args:
            word (unicode): word as IPA string

        Returns:
            unicode: longest single-segment prefix of `word`
        """
        match = self.seg_regex.match(word)
        if match:
            return match.group(0)
        else:
            return ''

    def seg_known(self, segment):
        """Return True if the segment is valid

        Args:
            segment (unicode): a string which may or may not be a valid segment

        Returns:
            bool: True if segment can be parsed given the database of bases and
                  diacritics
        """
        if self.seg_regex.match(segment):
            return True
        else:
            return False

    def filter_segs(self, segs):
        """Given list of strings, return only those which are valid segments.

        Args:
            segs (list): list of unicode values

        Returns:
            list: values in `segs` that are valid segments (according to the
                  definititions of bases and diacritics/modifiers known to the
                  object
        """
        def whole_seg(seg):
            m = self.seg_regex.match(seg)
            if m and m.group(0) == seg:
                return True
            else:
                return False
        return list(filter(whole_seg, segs))

    def segment_word_segments(self, word):
        def n2s(s):
            if s is None:
                return ''
            else:
                return s
        return ((n2s(m.group('pre')), n2s(m.group('base')), n2s(m.group('post')))
                for m in self.seg_regex.finditer(word))

    @property
    def all_segs_matching_fts(self):
        raise AttributeError("'PermissiveFeatureTable' object has no attribute 'all_segs_matching_fts'")