Vendor patched panphon (py3.8-compatible import)

panphon/__init__.py

@@ -0,0 +1,3 @@
+from __future__ import absolute_import
+from panphon.featuretable import FeatureTable
+from panphon._panphon import pat

panphon/_panphon.py

@@ -0,0 +1,544 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import, print_function, unicode_literals
+from os import stat
+import unicodedata
+
+import os.path
+from functools import reduce
+
+import numpy
+import pkg_resources
+
+import regex as re
+import unicodecsv as csv
+
+from panphon import featuretable
+
+from . import xsampa
+
+from panphon.errors import SegmentError
+
+# logging.basicConfig(level=logging.DEBUG)
+
+
+FT_REGEX = re.compile(r'([-+0])([a-z][A-Za-z]*)', re.U | re.X)
+MT_REGEX = re.compile(r'\[[-+0a-zA-Z ,;]*\]')
+SEG_REGEX = re.compile(r'[\p{InBasic_Latin}\p{InGreek_and_Coptic}' +
+                       r'\p{InIPA_Extensions}ŋœ\u00C0-\u00FF]' +
+                       r'[\u0300-\u0360\u0362-\u036F]*' +
+                       r'\p{InSpacing_Modifier_Letters}*',
+                       re.U | re.X)
+filenames = {
+    'spe+': os.path.join('data', 'ipa_all.csv'),
+    'panphon': os.path.join('data', 'ipa_all.csv'),
+}
+
+
+def segment_text(text, seg_regex=SEG_REGEX):
+    """Return an iterator of segments in the text.
+
+    Args:
+        text (unicode): string of IPA Unicode text
+        seg_regex (_regex.Pattern): compiled regex defining a segment (base +
+                                    modifiers)
+
+    Return:
+        generator: segments in the input text
+    """
+    for m in seg_regex.finditer(text):
+        yield m.group(0)
+
+
+def fts(s):
+    """Given string `s` with +/-[alphabetical sequence]s, return list of features.
+
+    Args:
+        s (str): string with segments of the sort "+son -syl 0cor"
+
+    Return:
+        list: list of (value, feature) tuples
+    """
+    return [m.groups() for m in FT_REGEX.finditer(s)]
+
+
+def pat(p):
+    """Given a string `p` with feature matrices (features grouped with square
+    brackets into segments, return a list of sets of (value, feature) tuples.
+
+    Args:
+        p (str): list of feature matrices as strings
+
+    Return:
+        list: list of sets of (value, feature) tuples
+    """
+    pattern = []
+    for matrix in [m.group(0) for m in MT_REGEX.finditer(p)]:
+        segment = set([m.groups() for m in FT_REGEX.finditer(matrix)])
+        pattern.append(segment)
+    return pattern
+
+
+def word2array(ft_names, word):
+    """Converts `word` [[(value, feature),...],...] to a NumPy array
+
+    Given a word consisting of lists of lists/sets of (value, feature) tuples,
+    return a NumPy array where each row is a segment and each column is a
+    feature.
+
+    Args:
+        ft_names (list): list of feature names (as strings) in order; this
+                         argument controls what features are included in the
+                         array that is output and their order vis-a-vis the
+                         columns of the array
+        word (list): list of lists of feature tuples (output by
+                     FeatureTable.word_fts)
+
+    Returns:
+        ndarray: array in which each row is a segment and each column
+                         is a feature
+    """
+    vdict = {'+': 1, '-': -1, '0': 0}
+
+    def seg2col(seg):
+        seg = dict([(k, v) for (v, k) in seg])
+        return [vdict[seg[ft]] for ft in ft_names]
+    return numpy.array([seg2col(s) for s in word], order='F')
+
+
+class FeatureTable(object):
+    """Encapsulate the segment <=> feature mapping in the file
+    "data/ipa_all.csv".
+    """
+
+    def __init__(self, feature_set='spe+'):
+        """Construct a FeatureTable object
+
+        Args:
+            feature_set (str): the feature set that the FeatureTable will use;
+                               currently, there is only one of these ("spe+")
+
+        """
+        filename = filenames[feature_set]
+        self.segments, self.seg_dict, self.names = self._read_table(filename)
+        self.seg_seq = {seg[0]: i for (i, seg) in enumerate(self.segments)}
+        self.weights = self._read_weights()
+        self.seg_regex = self._build_seg_regex()
+        self.longest_seg = max([len(x) for x in self.seg_dict.keys()])
+        self.xsampa = xsampa.XSampa()
+
+    @staticmethod
+    def normalize(data):
+        return unicodedata.normalize('NFD', data)
+
+    def _read_table(self, filename):
+        """Read the data from data/ipa_all.csv into self.segments, a
+        list of 2-tuples of unicode strings and sets of feature tuples and
+        self.seg_dict, a dictionary mapping from unicode segments and sets of
+        feature tuples.
+        """
+        filename = pkg_resources.resource_filename(
+            __name__, filename)
+        segments = []
+        with open(filename, 'rb') as f:
+            reader = csv.reader(f, encoding='utf-8')
+            header = next(reader)
+            names = header[1:]
+            for row in reader:
+                seg = row[0]
+                vals = row[1:]
+                specs = set(zip(vals, names))
+                segments.append((seg, specs))
+        seg_dict = dict(segments)
+        return segments, seg_dict, names
+
+    def _read_weights(self, filename=os.path.join('data', 'feature_weights.csv')):
+        filename = pkg_resources.resource_filename(
+            __name__, filename)
+        with open(filename, 'rb') as f:
+            reader = csv.reader(f, encoding='utf-8')
+            next(reader)
+            weights = [float(x) for x in next(reader)]
+        return weights
+
+    def _build_seg_regex(self):
+        # Build a regex that will match individual segments in a string.
+        segs = sorted(self.seg_dict.keys(), key=lambda x: len(x), reverse=True)
+        return re.compile(r'(?P<all>{})'.format('|'.join(segs)))
+
+    def fts(self, segment):
+        """Returns features corresponding to `segment` as list of (value,
+        feature) tuples.
+
+        Args:
+           segment (unicode): segment for which features are to be returned as
+                              Unicode IPA string.
+
+        Returns:
+            set: set of (value, feature) tuples, if `segment` is valid; otherwise,
+                 None
+        """
+        if segment in self.seg_dict:
+            return self.seg_dict[segment]
+        else:
+            return None
+
+    def match(self, ft_mask, ft_seg):
+        """Answer question "are `ft_mask`'s features a subset of ft_seg?"
+
+        Args:
+            ft_mask (set): pattern defined as set of (value, feature) tuples
+            ft_seg (set): segment defined as a set of (value, feature) tuples
+
+        Returns:
+            bool: True iff all features in `ft_mask` are also in `ft_seg`
+        """
+        return set(ft_mask) <= set(ft_seg)
+
+    def fts_match(self, features, segment):
+        """Answer question "are `ft_mask`'s features a subset of ft_seg?"
+
+        This is like `FeatureTable.match` except that it checks whether a
+        segment is valid and returns None if it is not.
+
+        Args:
+            features (set): pattern defined as set of (value, feature) tuples
+            segment (set): segment defined as a set of (value, feature) tuples
+
+        Returns:
+            bool: True iff all features in `ft_mask` are also in `ft_seg`; None
+                  if segment is not valid
+        """
+        features = set(features)
+        if self.seg_known(segment):
+            return features <= self.fts(segment)
+        else:
+            return None
+
+    def longest_one_seg_prefix(self, word, normalize=True):
+        """Return longest Unicode IPA prefix of a word
+
+        Args:
+            word (unicode): input word as Unicode IPA string
+
+        Returns:
+            unicode: longest single-segment prefix of `word` in database
+        """
+        if normalize:
+            word = FeatureTable.normalize(word)
+
+        for i in range(self.longest_seg, 0, -1):
+            if word[:i] in self.seg_dict:
+                return word[:i]
+        return ''
+
+    def validate_word(self, word):
+        """Returns True if `word` consists exhaustively of valid IPA segments
+
+        Args:
+            word (unicode): input word as Unicode IPA string
+
+        Returns:
+            bool: True if `word` can be divided exhaustively into IPA segments
+                  that exist in the database
+
+        """
+        while word:
+            match = self.seg_regex.match(word)
+            if match:
+                word = word[len(match.group(0)):]
+            else:
+                # print('{}\t->\t{}\t'.format(orig, word).encode('utf-8'), file=sys.stderr)
+                return False
+        return True
+
+    def segs(self, word):
+        """Returns a list of segments from a word
+
+        Args:
+            word (unicode): input word as Unicode IPA string
+
+        Returns:
+            list: list of strings corresponding to segments found in `word`
+        """
+        return [m.group('all') for m in self.seg_regex.finditer(word)]
+
+    def word_fts(self, word):
+        """Return featural analysis of `word`
+
+        Args:
+            word (unicode):  one or more IPA segments
+
+        Returns:
+            list: list of lists (value, feature) tuples where each inner list
+                  corresponds to a segment in `word`
+        """
+        return list(map(self.fts, self.segs(word)))
+
+    def word_array(self, ft_names, word):
+        """Return `word` as [-1, 0, 1] features in a NumPy array
+
+        Args:
+            ft_names (list): list of feature names in order
+            word (unicode): word as an IPA string
+
+        Returns:
+            ndarray: segments in rows, features in columns as [-1, 0 , 1]
+        """
+        return word2array(ft_names, self.word_fts(word))
+
+    def seg_known(self, segment):
+        """Return True if `segment` is in segment <=> features database
+
+        Args:
+            segment (unicode): consonant or vowel
+
+        Returns:
+            bool: True, if `segment` is in the database
+        """
+        return segment in self.seg_dict
+
+    def segs_safe(self, word):
+        """Return a list of segments (as strings) from a word
+
+        Characters that are not valid segments are included in the list as
+        individual characters.
+
+        Args:
+            word (unicode): word as an IPA string
+
+        Returns:
+            list: list of Unicode IPA strings corresponding to segments in
+                  `word`
+        """
+        segs = []
+        while word:
+            m = self.seg_regex.match(word)
+            if m:
+                segs.append(m.group(1))
+                word = word[len(m.group(1)):]
+            else:
+                segs.append(word[0])
+                word = word[1:]
+        return segs
+
+    def filter_segs(self, segs):
+        """Given list of strings, return only those which are valid segments
+
+        Args:
+            segs (list): list of IPA Unicode strings
+
+        Return:
+            list: list of IPA Unicode strings identical to `segs` but with
+                  invalid segments filtered out
+        """
+        return list(filter(self.seg_known, segs))
+
+    def filter_string(self, word):
+        """Return a string like the input but containing only legal IPA segments
+
+        Args:
+            word (unicode): input string to be filtered
+
+        Returns:
+            unicode: string identical to `word` but with invalid IPA segments
+                     absent
+
+        """
+        segs = [m.group(0) for m in self.seg_regex.finditer(word)]
+        return ''.join(segs)
+
+    def fts_intersection(self, segs):
+        """Return the features shared by `segs`
+
+        Args:
+            segs (list): list of Unicode IPA segments
+
+        Returns:
+            set: set of (value, feature) tuples shared by the valid segments in
+                 `segs`
+        """
+        fts_vecs = [self.fts(s) for s in self.filter_segs(segs)]
+        return reduce(lambda a, b: a & b, fts_vecs)
+
+    def fts_match_any(self, fts, inv):
+        """Return `True` if any segment in `inv` matches the features in `fts`
+
+        Args:
+            fts (list): a collection of (value, feature) tuples
+            inv (list): a collection of IPA segments represented as Unicode
+                        strings
+
+        Returns:
+            bool: `True` if any segment in `inv` matches the features in `fts`
+        """
+        return any([self.fts_match(fts, s) for s in inv])
+
+    def fts_match_all(self, fts, inv):
+        """Return `True` if all segments in `inv` matches the features in fts
+
+        Args:
+            fts (list): a collection of (value, feature) tuples
+            inv (list): a collection of IPA segments represented as Unicode
+                        strings
+
+        Returns:
+            bool: `True` if all segments in `inv` matches the features in `fts`
+        """
+        return all([self.fts_match(fts, s) for s in inv])
+
+    def fts_contrast2(self, fs, ft_name, inv):
+        """Return `True` if there is a segment in `inv` that contrasts in feature
+        `ft_name`.
+
+        Args:
+            fs (list): feature specifications used to filter `inv`.
+            ft_name (str): name of the feature where contrast must be present.
+            inv (list): collection of segments represented as Unicode segments.
+
+        Returns:
+            bool: `True` if two segments in `inv` are identical in features except
+                  for feature `ft_name`
+        """
+        inv_fts = [self.fts(x) for x in inv if set(fs) <= self.fts(x)]
+        for a in inv_fts:
+            for b in inv_fts:
+                if a != b:
+                    diff = a ^ b
+                    if len(diff) == 2:
+                        if all([nm == ft_name for (_, nm) in diff]):
+                            return True
+        return False
+
+    def fts_count(self, fts, inv):
+        """Return the count of segments in an inventory matching a given
+        feature mask.
+
+        Args:
+            fts (set): feature mask given as a set of (value, feature) tuples
+            inv (set): inventory of segments (as Unicode IPA strings)
+
+        Returns:
+            int: number of segments in `inv` that match feature mask `fts`
+        """
+        return len(list(filter(lambda s: self.fts_match(fts, s), inv)))
+
+    def match_pattern(self, pat, word):
+        """Implements fixed-width pattern matching.
+
+        Matches just in case pattern is the same length (in segments) as the
+        word and each of the segments in the pattern is a featural subset of the
+        corresponding segment in the word. Matches return the corresponding list
+        of feature sets; failed matches return None.
+
+        Args:
+           pat (list): pattern consisting of a sequence of sets of (value,
+                       feature) tuples
+           word (unicode): a Unicode IPA string consisting of zero or more
+                          segments
+
+        Returns:
+            list: corresponding list of feature sets or, if there is no match,
+                  None
+        """
+        segs = self.word_fts(word)
+        if len(pat) != len(segs):
+            return None
+        else:
+            if all([set(p) <= s for (p, s) in zip(pat, segs)]):
+                return segs
+
+    def match_pattern_seq(self, pat, const):
+        """Implements limited pattern matching. Matches just in case pattern is
+        the same length (in segments) as the constituent and each of the
+        segments in the pattern is a featural subset of the corresponding
+        segment in the word.
+
+        Args:
+            pat (list): pattern consisting of a list of sets of (value, feature)
+                        tuples.
+            const (list): a sequence of Unicode IPA strings consisting of zero
+                          or more segments.
+
+        Returns:
+            bool: `True` if `const` matches `pat`
+        """
+        segs = [self.fts(s) for s in const]
+        if len(pat) != len(segs):
+            return False
+        else:
+            return all([set(p) <= s for (p, s) in zip(pat, segs)])
+
+    def all_segs_matching_fts(self, fts):
+        """Return segments matching a feature mask, both as (value, feature)
+        tuples (sorted in reverse order by length).
+
+        Args:
+            fts (list): feature mask as (value, feature) tuples.
+
+        Returns:
+            list: segments matching `fts`, sorted in reverse order by length
+        """
+        matching_segs = []
+        for seg, pairs in self.segments:
+            if set(fts) <= set(pairs):
+                matching_segs.append(seg)
+        return sorted(matching_segs, key=lambda x: len(x), reverse=True)
+
+    def compile_regex_from_str(self, ft_str):
+        """Given a string describing features masks for a sequence of segments,
+        return a regex matching the corresponding strings.
+
+        Args:
+            ft_str (str): feature masks, each enclosed in square brackets, in
+            which the features are delimited by any standard delimiter.
+
+        Returns:
+           Pattern: regular expression pattern equivalent to `ft_str`
+        """
+
+        sequence = []
+        for m in re.finditer(r'\[([^]]+)\]', ft_str):
+            ft_mask = fts(m.group(1))
+            segs = self.all_segs_matching_fts(ft_mask)
+            sub_pat = '({})'.format('|'.join(segs))
+            sequence.append(sub_pat)
+        pattern = ''.join(sequence)
+        regex = re.compile(pattern)
+        return regex
+
+    def segment_to_vector(self, seg):
+        """Given a Unicode IPA segment, return a list of feature specificiations
+        in cannonical order.
+
+        Args:
+            seg (unicode): IPA consonant or vowel
+
+        Returns:
+            list: feature specifications ('+'/'-'/'0') in the order from
+            `FeatureTable.names`
+        """
+        ft_dict = {ft: val for (val, ft) in self.fts(seg)}
+        return [ft_dict[name] for name in self.names]
+
+    def tensor_to_numeric(self, t):
+        return list(map(lambda a:
+                    list(map(lambda b: {'+': 1, '-': -1, '0': 0}[b], a)), t))
+
+    def word_to_vector_list(self, word, numeric=False, xsampa=False):
+        """Return a list of feature vectors, given a Unicode IPA word.
+
+        Args:
+            word (unicode): string in IPA
+            numeric (bool): if True, return features as numeric values instead
+                            of strings
+
+        Returns:
+            list: a list of lists of '+'/'-'/'0' or 1/-1/0
+        """
+        if xsampa:
+            word = self.xsampa.convert(word)
+        tensor = list(map(self.segment_to_vector, self.segs(word)))
+        if numeric:
+            return self.tensor_to_numeric(tensor)
+        else:
+            return tensor

panphon/collapse.py

@@ -0,0 +1,38 @@
+from __future__ import (absolute_import, division, print_function,
+                        unicode_literals)
+
+import os.path
+
+import pkg_resources
+import yaml
+
+from panphon import _panphon
+from panphon import permissive
+
+
+class Collapser(object):
+    def __init__(self, tablename='dogolpolsky_prime.yml', feature_set='spe+', feature_model='strict'):
+        fm = {'strict': _panphon.FeatureTable,
+              'permissive': permissive.PermissiveFeatureTable}
+        self.fm = fm[feature_model](feature_set=feature_set)
+        self.rules = self._load_table(tablename)
+
+    def _load_table(self, tablename):
+        fn = os.path.join('data', tablename)
+        fn = pkg_resources.resource_filename(__name__, fn)
+        with open(fn, 'r') as f:
+            rules = []
+            table = yaml.load(f.read(), Loader=yaml.FullLoader)
+            for rule in table:
+                rules.append((_panphon.fts(rule['def']), rule['label']))
+        return rules
+
+    def collapse(self, s):
+        segs = []
+        for seg in self.fm.seg_regex.findall(s):
+            fts = self.fm.fts(seg)
+            for mask, label in self.rules:
+                if self.fm.match(mask, fts):
+                    segs.append(label)
+                    break
+        return ''.join(segs)

panphon/data/asjp.yml

@@ -0,0 +1,136 @@
+-
+    name: voiceless labiodental fricative
+    def: "[-son +cont -voi +strid +ant -cor +lab]"
+    label: f
+-
+    name: voiced labiodental fricative
+    def: "[-son +cont +voi +strid +ant -cor +lab]"
+    label: v
+-
+    name: bilabial nasal
+    def: "[+son +nas +ant -cor +lab]"
+    label: m
+-
+    name: voiceless bilabial obstruent
+    def: "[-son -voi +ant -cor +lab]"
+    label: p
+-
+    name: voiced bilabial obstruent
+    def: "[-son +voi +ant -cor +lab]"
+    label: b
+-
+    name: dental fricatives
+    def: "[-son +cont +ant +cor +dist]"
+    label: 8
+-
+    name: dental nasal
+    def: "[+son +nas -cont +ant +cor +dist]"
+    label: 4
+-
+    name: voiceless alveolar stop
+    def: "[-son -cont -voi +ant +cor -dist]"
+    label: t
+-
+    name: voiced alveolar stop
+    def: "[-son -cont +voi +ant +cor -dist]"
+    label: d
+-
+    name: voiceless alveolar fricative
+    def: "[-son -cont -voi +ant +cor -dist]"
+    label: s
+-
+    name: voiced alveolar fricative
+    def: "[-son -cont +voi +ant +cor -dist]"
+    label: z
+-
+    name: alveolar affricate
+    def: "[-son +delrel +ant +cor -dist]"
+    label: c
+-
+    name: alveolar nasal
+    def: "[+son -cont +nas +ant +cor -dist]"
+    label: n
+-
+    name: voiceless postalveolar fricative
+    def: "[-son +cont -voi -ant +cor +dist]"
+    label: S
+-
+    name: voiced postalveolar fricative
+    def: "[-son +cont +voi -ant +cor +dist]"
+    label: Z
+-
+    name: alveolar affricate
+    def: "[-son +delrel -voi -ant +cor +dist]"
+    label: C
+-
+    name: alveolar affricate
+    def: "[-son +delrel +voi -ant +cor +dist]"
+    label: j
+-
+    name: palatal stop
+    def: "[-son -cont -delrel -ant -cor +hi -lo -back]"
+    label: T
+-
+    name: palatal nasal
+    def: "[+son -cont +nas -ant -cor +hi -lo -back]"
+    label: 5
+-
+    name: voiceless velar stop
+    def: "[-son -cont -delrel -voi -ant -cor +hi -lo +back]"
+    label: k
+-
+    name: voiced velar stop
+    def: "[-son -cont -delrel +voi -ant -cor +hi -lo +back]"
+    label: g
+-
+    name: velar fricative
+    def: "[-son +cont -ant -cor +hi -lo +back]"
+    label: x
+-
+    name: velar nasal
+    def: "[+son -cont +nas -ant -cor +hi -lo +back]"
+    label: N
+-
+    name: voiceless uvular stop
+    def: "[-son -cont -delrel -voi -ant -cor -hi -lo +back]"
+    label: q
+-
+    name: voiced uvular stop
+    def: "[-son -cont -delrel +voi -ant -cor -hi -lo +back]"
+    label: G
+-
+    name: uvular and pharyngeal fricative
+    def: "[-son +cont -delrel -ant -cor -hi +back]"
+    label: X
+-
+    name: glottal stop
+    def: "[-syl +son -cont +cg -ant -cor -hi -lo -back]"
+    label: 7
+-
+    name: glottal fricative
+    def: "[-syl +son +cont -ant -cor -hi -lo -back]"
+    label: h
+-
+    name: coronal lateral
+    def: "[+son +cont +lat +ant +cor]"
+    label: l
+-
+    name: all non-coronal laterals
+    def: "[+son +cont +lat]"
+    label: L
+-
+    name: labiovelar approximant
+    def: "[-syl +son -cons +lab -ant -cor +hi -lo +back +round]"
+    label: w
+-
+    name: palatal approximant
+    def: "[-syl +son -cons -ant -cor +hi -lo -back]"
+    label: j
+-
+    name: coronal trill or tap
+    def: "[+son +cons +cont -lat +ant +cor]"
+    label: r
+-
+    name: coronal approximant
+    def: "[+son -cons +cont -lat +ant +cor]"
+    label: r

panphon/data/diacritic_definitions.yml

@@ -0,0 +1,669 @@
+# DIACRITICS AND MODIFIERS
+diacritics:
+
+# Airstream mechanism
+
+  - marker: ʼ
+    name: Ejective
+    position: post
+    conditions:
+      - son: "-"
+        voi: "-"
+    content:
+      cg: "+"
+
+# Laryngeal features
+
+  - marker: ̥
+    name: Voiceless
+    position: post
+    conditions:
+      - son: "+"
+        voi: "+"
+    content:
+      voi: "-"
+
+  - marker: ʰ
+    name: Aspirated
+    position: post
+    conditions:
+      - son: "-"
+        cg: "-"
+        cont: "-"
+      - cont: "+"
+        son: "-"
+        voi: "-"
+    content:
+      sg: "+"
+
+# Backness modifications
+
+  - marker: ̟
+    name: "Advanced"
+    position: post
+    conditions:
+      - syl: "-"
+      - syl: "+"
+    content: {}
+
+  - marker: ̠
+    name: "Retracted"
+    position: post
+    conditions:
+      - syl: "-"
+      - syl: "+"
+    content: {}
+
+  - marker: ̈
+    name: "Centralized"
+    position: post
+    conditions:
+      - syl: "+"
+    exclude:
+      - ə
+    content: {}
+
+# Syllabicity
+
+  - marker: ̩
+    name: "Syllabic"
+    position: post
+    conditions:
+      - syl: "-"
+        cont: "+"
+        delrel: "-"
+      - syl: "-"
+        son: "+"
+    exclude:
+      - ʔ
+    content:
+      syl: "+"
+
+  - marker: ̯
+    name: "Non-syllabic"
+    position: post
+    conditions:
+      - syl: "+"
+    content:
+      syl: "-"
+
+# Rhoticity
+
+  - marker: ˞
+    name: "Rhotacized"
+    position: post
+    conditions:
+      - syl: "+"
+    content:
+      ant: "-"
+      hi: "+"
+      round: "+"
+
+# Voice quality
+
+  - marker: ̤
+    name: "Breathy Voiced"
+    position: post
+    conditions:
+      - voi: "+"
+    exclude:
+      - ʔ
+    content:
+      sg: "+"
+
+  - marker: ̰
+    name: "Creaky Voiced"
+    position: post
+    conditions:
+      - voi: "+"
+    exclude:
+      - ʔ
+    content:
+      cg: "+"
+
+  - marker: ˀ # not IPA
+    name: Glottalized
+    position: post
+    conditions:
+      - cg: "-"
+    content:
+      cg: "+"
+
+  - marker: ˀ # not IPA
+    name: Preglottalized
+    position: pre
+    conditions:
+      - syl: "-"
+      - cg: "-"
+    content:
+      cg: "+"
+
+# Secondary articulations
+
+  - marker: ̼
+    name: Linguolabial
+    position: post
+    conditions:
+      - cor: "+"
+        ant: "+"
+        cont: "-"
+        delrel: "-"
+      - cont: "+"
+        son: "-"
+        cor: "+"
+        ant: "+"
+        strid: "-"
+        delrel: "-"
+    content:
+      lab: "+"
+
+  - marker: ʷ
+    name: Labialized
+    position: post
+    conditions:
+      - syl:  "-"
+    exclude:
+      - w
+      - ʍ
+      - ɥ
+    content:
+      round: "+"
+      back:  "+"
+      hi:  "+"
+
+  - marker: ʲ
+    name: Palatalized
+    position: post
+    conditions:
+      - syl: "-"
+    exclude:
+      - j
+      - ɥ
+    content:
+      hi:   "+"
+      back: "-"
+
+  - marker: ᶣ
+    name: Labiopalatalized
+    position: post
+    conditions:
+      - lab: "-"
+        syl: "-"
+    content:
+      hi: "+"
+      back: "-"
+      round: "+"
+
+  - marker: ˠ
+    name: Velarized
+    position: post
+    conditions:
+      - syl: "-"
+      - hi: "-"
+      - back: "-"
+    exclude:
+      - k
+      - ɡ
+      - ŋ
+      - x
+      - ɣ
+      - ɰ
+      - ʟ
+    content:
+      hi:   "+"
+      back: "+"
+
+  - marker: ˤ
+    name: Pharyngealized
+    position: post
+    conditions:
+      - {}
+    exclude:
+      - ʔ
+      - ʕ
+      - ħ
+    content:
+      lo:   "+"
+      back: "+"
+
+  - marker: ̴
+    name: "Velarized or Pharyngealized"
+    position: post
+    conditions:
+      - cor: "+"
+        lat: "+"
+        delrel: "-"
+    content:
+      hi:   "+"
+      back: "+"
+
+# Height modifications
+
+  - marker: ̝
+    name: Raised
+    position: post
+    conditions:
+      - cont: "+"
+    content: {}
+
+  - marker: ̞
+    name: Lowered
+    position: post
+    conditions:
+      - cont: "+"
+    content: {}
+
+# Tongue root state
+
+  - marker: ̘
+    name: ATR
+    position: post
+    conditions:
+      - syl: "+"
+    content:
+      tense: "+"
+
+  - marker: ̙
+    name: RTR
+    position: post
+    conditions:
+      - syl: "+"
+    content:
+      tense: "-"
+
+# Coronal modifications
+
+  - marker: ̺
+    name: Apical
+    position: post
+    conditions:
+      - cor: "+"
+    content:
+      distr: "-"
+
+  - marker: ̻
+    name: Laminal
+    position: post
+    conditions:
+      - cor: "+"
+    content:
+      distr: "+"
+
+# Nasality
+
+  - marker: ̃
+    name: Nasalized
+    position: post
+    conditions:
+      - voi: "+"
+        nas: "-"
+    exclude:
+      - ʔ
+      - h
+    content:
+      nas: "+"
+
+# Properties of release
+
+  - marker: ⁿ
+    name: "Nasal Release"
+    position: post
+    conditions:
+      - son: "-"
+        cont: "-"
+        delrel: "-"
+    content:
+      nas: "+"
+
+  - marker: ˡ
+    name: "Lateral Release"
+    position: post
+    conditions:
+      - son: "-"
+        cont: "-"
+        cor: "+"
+        delrel: "-"
+    content:
+      lat: "+"
+      delrel: "+"
+
+# Suprasegmentals
+
+  - marker: ː
+    name: Long
+    position: post
+    conditions:
+      - long: "-"
+    content:
+      long: "+"
+
+  - marker: ̆
+    name: Extra Short
+    position: post
+    conditions:
+      - syl: "+"
+    content:
+      long: "-"
+
+# COMBINATIONS OF DIACRITICS AND MODIFIERS
+combinations:
+
+# Quantity
+
+  - name: Long Labialized
+    combines:
+    - Long
+    - Labialized
+
+  - name: Long Palatalized
+    combines:
+    - Long
+    - Palatalized
+
+  - name: Long Velarized
+    combines:
+    - Long
+    - Velarized
+
+  - name: Long Pharyngealized
+    combines:
+    - Long
+    - Pharyngealized
+
+# Airstream mechanisms
+
+## Ejective
+
+  - name: Ejective Labialized
+    combines:
+    - Ejective
+    - Labialized
+
+  - name: Ejective Palatalized
+    combines:
+    - Ejective
+    - Palatalized
+
+  - name: Ejective Long
+    combines:
+    - Ejective
+    - Long
+
+# Laryngeal features
+
+## Voiceless
+
+  - name: Voiceless Labialized
+    combines:
+    - Voiceless
+    - Labialized
+
+  - name: Voiceless Palatalized
+    combines:
+    - Voiceless
+    - Palatalized
+
+  - name: Voiceless Velarized
+    combines:
+    - Voiceless
+    - Velarized
+
+  - name: Voiceless Pharyngealized
+    combines:
+    - Voiceless
+    - Pharyngealized
+
+  - name: Voiceless Long
+    combines:
+    - Voiceless
+    - Long
+
+## Aspirated
+
+  - name: Aspirated Labialized
+    combines:
+    - Aspirated
+    - Labialized
+
+  - name: Aspirated Palatalized
+    combines:
+    - Aspirated
+    - Palatalized
+
+  - name: Aspirated Velarized
+    combines:
+    - Aspirated
+    - Velarized
+
+  - name: Aspirated Labiopalatalized
+    combines:
+    - Aspirated
+    - Labiopalatalized
+
+  - name: Aspirated Pharyngealized
+    combines:
+    - Aspirated
+    - Pharyngealized
+
+  - name: Aspirated Long
+    combines:
+    - Aspirated
+    - Long
+
+# Syllabicity
+
+  - name: Syllabic Labialized
+    combines:
+    - Syllabic
+    - Labialized
+
+# Rhoticity
+
+# Voice quality
+
+## Breathy voice
+
+  - name: Breathy Devoiced
+    combines:
+    - "Breathy Voiced"
+    - Voiceless
+
+  - name: Breathy Labialized
+    combines:
+    - "Breathy Voiced"
+    - Labialized
+
+  - name: Breathy Palatalized
+    combines:
+    - "Breathy Voiced"
+    - Palatalized
+
+  - name: Breathy Velarized
+    combines:
+    - "Breathy Voiced"
+    - Velarized
+
+  - name: Breathy Pharyngealized
+    combines:
+    - "Breathy Voiced"
+    - Pharyngealized
+
+  - name: Breathy Long
+    combines:
+    - "Breathy Voiced"
+    - Long
+
+## Creaky voice
+
+  - name: Creaky Labialized
+    combines:
+    - Creaky Voiced
+    - Labialized
+
+  - name: Creaky Palatalized
+    combines:
+    - Creaky Voiced
+    - Palatalized
+
+  - name: Creaky Velarized
+    combines:
+    - Creaky Voiced
+    - Velarized
+
+  - name: Creaky Pharyngealized
+    combines:
+    - Creaky Voiced
+    - Pharyngealized
+
+  - name: Creaky Long
+    combines:
+    - Creaky Voiced
+    - Long
+
+# Secondary articulations
+
+## Linguolabial
+
+## Labialized
+
+  - name: Labialized Aspirated
+    combines:
+    - Labialized
+    - Aspirated
+
+  - name: Labialized Aspirated Long
+    combines:
+    - Labialized
+    - Aspirated
+    - Long
+
+  - name: Labialized Ejective
+    combines:
+    - Labialized
+    - Ejective
+
+  - name: Labialized Glottalized
+    combines:
+    - Labialized
+    - Glottalized
+
+  - name: Labialized Velarized
+    combines:
+    - Labialized
+    - Velarized
+
+  - name: Labialized Velarized Aspirated
+    combines:
+    - Labialized
+    - Velarized
+    - Aspirated
+
+  - name: Labialized Pharyngealized
+    combines:
+    - Labialized
+    - Pharyngealized
+
+  - name: Labialized Pharyngealized Ejective
+    combines:
+    - Labialized
+    - Pharyngealized
+    - Ejective
+
+  - name: Labialized Pharyngealized Aspirated
+    combines:
+    - Labialized
+    - Pharyngealized
+    - Aspirated
+
+  - name: Labialized Long
+    combines:
+    - Labialized
+    - Long
+
+## Palatalized
+
+  - name: Palatalized Ejective
+    combines:
+    - Palatalized
+    - Ejective
+
+  - name: Palatalized Labialized
+    combines:
+    - Palatalized
+    - Labialized
+
+  - name: Palatalized Long
+    combines:
+    - Palatalized
+    - Long
+
+  - name: Palatalized Aspirated
+    combines:
+    - Palatalized
+    - Aspirated
+
+  - name: Palatalized Labialized Aspirated
+    combines:
+    - Palatalized
+    - Labialized
+    - Aspirated
+
+# Pharyngealized
+
+  - name: Pharyngealized Ejective
+    combines:
+    - Pharyngealized
+    - Ejective
+
+  - name: Pharyngealized Aspirated
+    combines:
+    - Pharyngealized
+    - Aspirated
+
+  - name: Pharyngealized Long
+    combines:
+    - Pharyngealized
+    - Long
+
+ # Tongue root state
+
+## ATR
+
+  - name: ATR Velarized
+    combines:
+    - ATR
+    - Velarized
+
+  - name: ATR Long
+    combines:
+    - ATR
+    - Long
+
+## RTR
+
+  - name: RTR Velarized
+    combines:
+    - RTR
+    - Velarized
+
+  - name: RTR Long
+    combines:
+    - RTR
+    - Long
+
+# Nasality
+
+  - name: Nasalized Creaky
+    combines:
+    - Nasalized
+    - Creaky Voiced
+
+  - name: Nasalized Pharyngealized
+    combines:
+    - Nasalized
+    - Pharyngealized
+
+  - name: Nasalized Long
+    combines:
+    - Nasalized
+    - Long

panphon/data/diacritic_definitions_schema.yml

@@ -0,0 +1,209 @@
+type: map
+mapping:
+  "diacritics":
+    type: seq
+    sequence:
+      - type: map
+        mapping:
+          "marker":
+            type: str
+            required: yes
+          "name":
+            type: str
+            required: yes
+          "position":
+            type: str
+            required: no
+          "conditions":
+            type: seq
+            sequence:
+              - type: map
+                mapping:
+                  "syl":
+                    type: str
+                    enum: ["0", "-", "+"]
+                    required: no
+                  "son":
+                    type: str
+                    enum: ["0", "-", "+"]
+                    required: no
+                  "cons":
+                    type: str
+                    enum: ["0", "-", "+"]
+                    required: no
+                  "cont":
+                    type: str
+                    enum: ["0", "-", "+"]
+                    required: no
+                  "delrel":
+                    type: str
+                    enum: ["0", "-", "+"]
+                    required: no
+                  "lat":
+                    type: str
+                    enum: ["0", "-", "+"]
+                    required: no
+                  "nas":
+                    type: str
+                    enum: ["0", "-", "+"]
+                    required: no
+                  "strid":
+                    type: str
+                    enum: ["0", "-", "+"]
+                    required: no
+                  "voi":
+                    type: str
+                    enum: ["0", "-", "+"]
+                    required: no
+                  "sg":
+                    type: str
+                    enum: ["0", "-", "+"]
+                    required: no
+                  "cg":
+                    type: str
+                    enum: ["0", "-", "+"]
+                    required: no
+                  "ant":
+                    type: str
+                    enum: ["0", "-", "+"]
+                    required: no
+                  "cor":
+                    type: str
+                    enum: ["0", "-", "+"]
+                    required: no
+                  "distr":
+                    type: str
+                    enum: ["0", "-", "+"]
+                    required: no
+                  "lab":
+                    type: str
+                    enum: ["0", "-", "+"]
+                    required: no
+                  "hi":
+                    type: str
+                    enum: ["0", "-", "+"]
+                    required: no
+                  "lo":
+                    type: str
+                    enum: ["0", "-", "+"]
+                    required: no
+                  "back":
+                    type: str
+                    enum: ["0", "-", "+"]
+                    required: no
+                  "round":
+                    type: str
+                    enum: ["0", "-", "+"]
+                    required: no
+                  "tense":
+                    type: str
+                    enum: ["0", "-", "+"]
+                    required: no
+                  "long":
+                    type: str
+                    enum: ["0", "-", "+"]
+                    required: no
+          "exclude":
+            type: seq
+            sequence:
+              - type: str
+          "content":
+            type: map
+            mapping:
+              "syl":
+                type: str
+                enum: ["0", "-", "+"]
+                required: no
+              "son":
+                type: str
+                enum: ["0", "-", "+"]
+                required: no
+              "cons":
+                type: str
+                enum: ["0", "-", "+"]
+                required: no
+              "cont":
+                type: str
+                enum: ["0", "-", "+"]
+                required: no
+              "delrel":
+                type: str
+                enum: ["0", "-", "+"]
+                required: no
+              "lat":
+                type: str
+                enum: ["0", "-", "+"]
+                required: no
+              "nas":
+                type: str
+                enum: ["0", "-", "+"]
+                required: no
+              "strid":
+                type: str
+                enum: ["0", "-", "+"]
+                required: no
+              "voi":
+                type: str
+                enum: ["0", "-", "+"]
+                required: no
+              "sg":
+                type: str
+                enum: ["0", "-", "+"]
+                required: no
+              "cg":
+                type: str
+                enum: ["0", "-", "+"]
+                required: no
+              "ant":
+                type: str
+                enum: ["0", "-", "+"]
+                required: no
+              "cor":
+                type: str
+                enum: ["0", "-", "+"]
+                required: no
+              "distr":
+                type: str
+                enum: ["0", "-", "+"]
+                required: no
+              "lab":
+                type: str
+                enum: ["0", "-", "+"]
+                required: no
+              "hi":
+                type: str
+                enum: ["0", "-", "+"]
+                required: no
+              "lo":
+                type: str
+                enum: ["0", "-", "+"]
+                required: no
+              "back":
+                type: str
+                enum: ["0", "-", "+"]
+                required: no
+              "round":
+                type: str
+                enum: ["0", "-", "+"]
+                required: no
+              "tense":
+                type: str
+                enum: ["0", "-", "+"]
+                required: no
+              "long":
+                type: str
+                enum: ["0", "-", "+"]
+                required: no
+
+  "combinations":
+    type: seq
+    sequence:
+      - type: map
+        mapping:
+          "name":
+            type: str
+            required: yes
+          "combines":
+            type: seq
+            sequence:
+              - type: str

panphon/data/dolgopolsky_prime.yml

@@ -0,0 +1,56 @@
+-
+    name: labial obstruents
+    def: "[-son +ant -cor + lab]"
+    label: P
+-
+    name: coronal fricatives
+    def: "[-son +cor +cont +strid -delrel]"
+    label: S
+-
+    name: velar/postvelar obstruents
+    def: "[-son -ant -cor +back]"
+    label: K
+-
+    name: coronal affricates
+    def: "[+cor +delrel]"
+    label: K
+-
+    name: other coronal obstruents
+    def: "[-son +cor]"
+    label: T
+-
+    name: labial nasal
+    def: "[-syl +son +nas +ant -cor +lab]"
+    label: M
+-
+    name: nasal
+    def: "[-syl +son -cont +nas]"
+    label: N
+-
+    name: lateral
+    def: "[+lat]"
+    label: R
+-
+    name: approximates and trills
+    def: "[-syl +son -nas]"
+    label: R
+-
+    name: palatal glide
+    def: "[-syl +son +cont +hi -lo -back]"
+    label: J
+-
+    name: labiovelar glide
+    def: "[-syl +son]"
+    label: W
+-
+    name: laryngeals
+    def: "[-syl +son +cons +cont -cor -hi -back -lo]"
+    label: H
+-
+    name: consonants
+    def: "[-syl]"
+    label: C
+-
+    name: vowels
+    def: "[+syl]"
+    label: V

panphon/data/feature_weights.csv

@@ -0,0 +1,2 @@
+syl,son,cons,cont,delrel,lat,nas,strid,voi,sg,cg,ant,cor,distr,lab,hi,lo,back,round,tense,long,velaric
+1,1,1,0.5,0.25,0.25,0.25,0.125,0.125,0.125,0.125,0.25,0.25,0.125,0.25,0.25,0.25,0.25,0.25,0.25,0.125,0.25

panphon/data/ipa-xsampa.csv

@@ -0,0 +1,175 @@
+IPA,X-SAMPA,Name
+p,p,vl bilabial plosive
+b,b,vd bilabial plosive
+t,t,vl alveolar plosive
+d,d,vd alveolar plosive
+ʈ,t`,vl retroflex plosive
+ɖ,d`,vd retroflex plosive
+c,c,vl palatal plosive
+ɟ,J\,vd palatal plosive
+k,k,ld velar plosive
+ɡ,g,vd velar plosive
+q,q,vl uvular plosive
+ɢ,G\,vd uvular plosive
+ʔ,?,glottal plosive
+m,m,bilabial nasal
+ɱ,F,vl labiodental nasal
+n,n,alveolar nasal
+ɳ,n`,vl retroflex nasal
+ɲ,J,vl palatal nasal
+ŋ,N,vl velar nasal
+ɴ,N\,vl uvular nasal
+ʙ,B\,vd bilabial trill
+r,r,vd alveolar trill
+ʀ,R\,vl uvular trill
+ɾ,4,vl alveolar tap
+ɽ,r`,vl retroflex flap
+ɸ,p\,vl bilabial fricative
+β,B,vd bilabial fricative
+f,f,vl labiodental fricative
+v,v,vd labiodental fricative
+θ,T,vl dental fricative
+ð,D,vd dental fricative
+s,s,vl alveolar fricative
+z,z,vd alveolar fricative
+ʃ,S,vl postalveolar fricative
+ʒ,Z,vd postalveolar fricative
+ʂ,s`,vl retroflex fricative
+ʐ,z`,vd retroflex fricative
+ç,C,vl palatal fricative
+ʝ,j\,vd palatal fricative
+x,x,vl velar fricative
+ɣ,G,vd velar fricative
+χ,X,vl uvular fricative
+ʁ,R,vd uvular fricative
+ħ,X\,vl pharyngeal fricative
+ʕ,?\,vd pharyngeal fricative
+h,h,vl glottal fricative
+ʔ,?,glottal plosive
+ɬ,K,vl alveolar lateral fricative
+ɮ,K\,vd alveolar lateral fricative
+ʋ,P,vd labiodental approximant
+ɹ,r\,vd (post)alveolar approximant
+ɻ,r\`,vd retroflex approximant
+j,j,vd palatal approximant
+ɰ,M\,vd velar approximant
+l,l,vd alveolar lateral approximant
+ɭ,l`,vd retroflex lateral approximant
+ʎ,L,vd palatal lateral approximant
+ʟ,L\,vd velar lateral approximant
+pʼ,p_>,ejective
+tʼ,t_>,ejective
+ʈʼ,t`_>,ejective
+cʼ,c_>,ejective
+kʼ,k_>,ejective
+qʼ,q_>,ejective
+ɓ,b_<,vl bilabial implosive
+ɗ,d_<,vl alveolar implosive
+ƙ,k_<,vl velar implosive
+ɠ,g_<,vl velar implosive
+i,i,close front unrounded
+y,y,close front rounded
+ɨ,1,close central unrounded
+ʉ,},close central rounded
+ɯ,M,close back unrounded
+u,u,close back rounded
+ɪ,I,lax close front unrounded
+ʏ,Y,lax close front rounded
+ʊ,U,lax close back rounded
+e,e,close-mid front unrounded
+ø,2,front close-mid rounded
+ɤ,7,close-mid back unrounded
+o,o,close-mid back rounded
+ə,@,schwa
+ɘ,@\,close-mid central unrounded vowel
+ɵ,8,rounded schwa
+ɛ,E,open-mid front unrounded
+œ,9,front open-mid rounded
+ʌ,V,open-mid back unrounded
+ɔ,O,open-mid back rounded
+æ,{,mid-open front unrounded vowel
+ɐ,6,open-mid schwa
+a,a,open front unrounded
+ă,a_X,extra short open front unrounded
+ɶ,&,front open rounded
+ɑ,A,open back unrounded
+ɒ,Q,open back rounded
+̥,_0,voiceless
+̬,_v,voiced
+ʰ,_h,aspirated
+̤,_t,breathy voiced
+̰,_k,creaky voiced
+̼,_N,linguolabial
+̪,_d,dental
+̺,_a,apical
+̻,_m,laminal
+̹,_O,more rounded
+̜,_c,less rounded
+̟,_+,advanced
+̠,_-,retracted
+̈,"_""",centralized
+̽,_x,mid-centralized
+̩,=,syllabic
+̯,_^,non-syllabic
+ʷ,_w,labialized
+ʲ,',palatalized
+ˠ,_G,velarized
+ˤ,_?\,pharyngealized
+̴,_e,velarized or pharyngealized
+̝,_r,raised
+̞,_o,lowered
+̃,~,nasalized
+ⁿ,_n,nasal release
+ˡ,_l,lateral release
+̚,_},not audibly released
+̘,_A,advanced tongue root
+̙,_q,retracted tongue root
+̋,_T,extra high tone
+́,_H,high tone
+̄,_M,mid tone
+̀,_L,low tone
+̏,_B,extra low tone
+ˈ,"""",(primary) stress mark
+ˌ,%,secondary stress
+ː,:,length mark
+ˑ,:\,half-length
+̆,_X,extra-short
+.,.,syllable break
+ʍ,W,vl labial-velar fricative
+w,w,vd labio-velar approximant
+ɥ,H,labial-palatal approximant
+ʜ,H\,vl epiglottal fricative
+ʢ,<\,vl epiglottal fricative
+ʡ,>\,vl epiglottal plosive
+ɕ,s\,vl alveolopalatal fricative
+ʑ,z\,vl alveolopalatal fricative
+ʘ,O\,bilabial click
+ǀ,|\,dental click
+ǃ,!\,click
+ǂ,'=\,alveolar click
+ǁ,|\|\,alveolar lateral click
+ɺ,l\,vl alveolar lateral flap
+ɜ,3,open-mid central
+ʛ,G\_<,vl uvular implosive
+ɚ,@`,rhotacized schwa
+ɞ,3\,open-mid central rounded
+ɦ,h\,vd glottal fricative
+ɫ,5,velarized vl alveolar lateral
+ʄ,J\_<,vl palatal implosive
+ʼ,_>,ejective
+ɝ,3`,rhotacized open-mid central
+t͡ʃ,tS,vl postalveolar affricate
+d͡ʒ,dZ,vd postalveolar affricate
+t͡ɕ,ts\,vl alveolo-palatal affricate
+d͡ʑ,dz\,vd alveolo-palatal affricate
+t͡ɬ,tK,vl alveolar lateral affricate
+k͡p,kp,vl labial-velar plosive
+g͡b,gb,vd labial-velar plosive
+ŋ͡m,Nm,labial-velar nasal stop
+ʈ͡ʂ,ts`,vl retroflex affricate
+ɖ͡ʐ,tz`,vd retroflex affricate
+˩,_B,extra low tone
+˨,_L,low tone
+˧,_M,mid tone 
+˦,_H,high tone
+˥,_T,extra high tone

panphon/data/ipa_bases.csv

@@ -0,0 +1,151 @@
+ipa,syl,son,cons,cont,delrel,lat,nas,strid,voi,sg,cg,ant,cor,distr,lab,hi,lo,back,round,velaric,tense,long,hitone,hireg
+ʘ,-,-,+,-,-,-,-,0,-,-,-,0,-,0,+,+,-,0,-,+,0,-,0,0
+ǀ,-,-,+,-,-,-,-,0,-,-,-,+,+,+,-,+,-,0,-,+,0,-,0,0
+ǃ,-,-,+,-,-,-,-,0,-,-,-,+,+,-,-,+,-,0,-,+,0,-,0,0
+ǂ,-,-,+,-,-,-,-,0,-,-,-,-,+,+,-,+,-,0,-,+,0,-,0,0
+ǁ,-,-,+,-,-,+,-,0,-,-,-,+,+,-,-,+,-,0,-,+,0,-,0,0
+k͡p,-,-,+,-,-,-,-,0,-,-,-,0,-,0,+,+,-,0,-,-,0,-,0,0
+ɡ͡b,-,-,+,-,-,-,-,0,+,-,-,0,-,0,+,+,-,0,-,-,0,-,0,0
+c,-,-,+,-,-,-,-,0,-,-,-,-,-,0,-,+,-,-,-,-,0,-,0,0
+ɡ,-,-,+,-,-,-,-,0,+,-,-,-,-,0,-,+,-,+,-,-,0,-,0,0
+k,-,-,+,-,-,-,-,0,-,-,-,-,-,0,-,+,-,+,-,-,0,-,0,0
+q,-,-,+,-,-,-,-,0,-,-,-,-,-,0,-,-,-,+,-,-,0,-,0,0
+ɖ,-,-,+,-,-,-,-,0,+,-,-,-,+,-,-,-,-,-,-,-,0,-,0,0
+ɟ,-,-,+,-,-,-,-,0,+,-,-,-,-,0,-,+,-,-,-,-,0,-,0,0
+ɠ,-,-,+,-,-,-,-,0,+,-,+,-,-,0,-,+,-,+,-,-,0,-,0,0
+ɢ,-,-,+,-,-,-,-,0,+,-,-,-,-,0,-,-,-,+,-,-,0,-,0,0
+ʄ,-,-,+,-,-,-,-,0,+,-,+,-,-,0,-,+,-,-,-,-,0,-,0,0
+ʈ,-,-,+,-,-,-,-,0,-,-,-,-,+,-,-,-,-,-,-,-,0,-,0,0
+ʛ,-,-,+,-,-,-,-,0,+,-,+,-,-,0,-,-,-,+,-,-,0,-,0,0
+b,-,-,+,-,-,-,-,0,+,-,-,+,-,0,+,-,-,-,-,-,0,-,0,0
+b͡d,-,-,+,-,-,-,-,0,+,-,-,+,+,-,+,-,-,-,-,-,0,-,0,0
+d,-,-,+,-,-,-,-,0,+,-,-,+,+,-,-,-,-,-,-,-,0,-,0,0
+d̪,-,-,+,-,-,-,-,0,+,-,-,+,+,+,-,-,-,-,-,-,0,-,0,0
+p,-,-,+,-,-,-,-,0,-,-,-,+,-,0,+,-,-,-,-,-,0,-,0,0
+p͡t,-,-,+,-,-,-,-,0,-,-,-,+,+,-,+,-,-,-,-,-,0,-,0,0
+t,-,-,+,-,-,-,-,0,-,-,-,+,+,-,-,-,-,-,-,-,0,-,0,0
+t̪,-,-,+,-,-,-,-,0,-,-,-,+,+,+,-,-,-,-,-,-,0,-,0,0
+ɓ,-,-,+,-,-,-,-,0,+,-,+,+,-,0,+,-,-,-,-,-,0,-,0,0
+ɗ,-,-,+,-,-,-,-,0,+,-,+,+,+,-,-,-,-,-,-,-,0,-,0,0
+b͡β,-,-,+,-,+,-,-,-,+,-,-,+,-,0,+,0,0,0,-,-,0,-,0,0
+k͡x,-,-,+,-,+,-,-,0,-,-,-,-,-,0,-,+,-,0,-,-,0,-,0,0
+p͡ɸ,-,-,+,-,+,-,-,-,-,-,-,+,-,0,+,0,0,0,-,-,0,-,0,0
+q͡χ,-,-,+,-,+,-,-,0,-,-,-,-,-,0,-,-,-,+,-,-,0,-,0,0
+ɡ͡ɣ,-,-,+,-,+,-,-,0,+,-,-,-,-,0,-,+,-,0,-,-,0,-,0,0
+ɢ͡ʁ,-,-,+,-,+,-,-,0,+,-,-,-,-,0,-,-,-,+,-,-,0,-,0,0
+c͡ç,-,-,+,-,+,-,-,0,-,-,-,-,+,+,-,+,-,-,-,-,0,-,0,0
+d͡ʒ,-,-,+,-,+,-,-,+,+,-,-,-,+,+,-,-,-,-,-,-,0,-,0,0
+t͡ʃ,-,-,+,-,+,-,-,+,-,-,-,-,+,+,-,-,-,-,-,-,0,-,0,0
+ɖ͡ʐ,-,-,+,-,+,-,-,+,+,-,-,-,+,-,-,0,0,0,-,-,0,-,0,0
+ɟ͡ʝ,-,-,+,-,+,-,-,0,+,-,-,-,+,+,-,+,-,-,-,-,0,-,0,0
+ʈ͡ʂ,-,-,+,-,+,-,-,+,-,-,-,-,+,-,-,0,0,0,-,-,0,-,0,0
+b͡v,-,-,+,-,+,-,-,+,+,-,-,+,-,0,+,-,-,-,-,-,0,-,0,0
+d̪͡z̪,-,-,+,-,+,-,-,+,+,-,-,+,+,+,-,0,0,0,-,-,0,-,0,0
+d̪͡ð,-,-,+,-,+,-,-,0,+,-,-,+,+,+,-,0,0,0,-,-,0,-,0,0
+d̪͡ɮ̪,-,-,+,-,+,+,-,0,+,-,-,+,+,-,-,0,0,0,-,-,0,-,0,0
+d͡z,-,-,+,-,+,-,-,+,+,-,-,+,+,-,-,-,-,-,-,-,0,-,0,0
+d͡ɮ,-,-,+,-,+,+,-,0,+,-,-,-,+,-,-,0,0,0,-,-,0,-,0,0
+d͡ʑ,-,-,+,-,+,-,-,0,+,-,-,-,+,+,-,+,-,-,-,-,0,-,0,0
+p͡f,-,-,+,-,+,-,-,+,-,-,-,+,-,0,+,-,-,-,-,-,0,-,0,0
+t̪͡s̪,-,-,+,-,+,-,-,+,-,-,-,+,+,+,-,0,0,0,-,-,0,-,0,0
+t̪͡ɬ̪,-,-,+,-,+,+,-,0,-,-,-,+,+,+,-,0,0,0,-,-,0,-,0,0
+t̪͡θ,-,-,+,-,+,-,-,0,-,-,-,+,+,+,-,0,0,0,-,-,0,-,0,0
+t͡s,-,-,+,-,+,-,-,+,-,-,-,+,+,-,-,-,-,-,-,-,0,-,0,0
+t͡ɕ,-,-,+,-,+,-,-,0,-,-,-,-,+,+,-,+,-,-,-,-,0,-,0,0
+t͡ɬ,-,-,+,-,+,+,-,0,-,-,-,+,+,-,-,0,0,0,-,-,0,-,0,0
+x,-,-,+,+,-,-,-,0,-,-,-,-,-,0,-,+,-,+,-,-,0,-,0,0
+ç,-,-,+,+,-,-,-,0,-,-,-,-,-,0,-,+,-,-,-,-,0,-,0,0
+ħ,-,-,+,+,-,-,-,0,-,-,-,-,-,0,-,-,+,+,-,-,0,-,0,0
+ɣ,-,-,+,+,-,-,-,0,+,-,-,-,-,0,-,+,-,+,-,-,0,-,0,0
+ʁ,-,-,+,+,-,-,-,0,+,-,-,-,-,0,-,-,-,+,-,-,0,-,0,0
+ʂ,-,-,+,+,-,-,-,+,-,-,-,-,+,-,-,-,-,-,-,-,0,-,0,0
+ʃ,-,-,+,+,-,-,-,+,-,-,-,-,+,+,-,-,-,-,-,-,0,-,0,0
+ʐ,-,-,+,+,-,-,-,+,+,-,-,-,+,-,-,-,-,-,-,-,0,-,0,0
+ʒ,-,-,+,+,-,-,-,+,+,-,-,-,+,+,-,-,-,-,-,-,0,-,0,0
+ʕ,-,-,+,+,-,-,-,0,+,-,-,-,-,0,-,-,+,+,-,-,0,-,0,0
+ʝ,-,-,+,+,-,-,-,0,+,-,-,-,-,0,-,+,-,-,-,-,0,-,0,0
+χ,-,-,+,+,-,-,-,0,-,-,-,-,-,0,-,-,-,+,-,-,0,-,0,0
+f,-,-,+,+,-,-,-,+,-,-,-,+,-,0,+,-,-,-,-,-,0,-,0,0
+s,-,-,+,+,-,-,-,+,-,-,-,+,+,-,-,-,-,-,-,-,0,-,0,0
+s̪,-,-,+,+,-,-,-,+,-,-,-,+,+,+,-,-,-,-,-,-,0,-,0,0
+v,-,-,+,+,-,-,-,+,+,-,-,+,-,0,+,-,-,-,-,-,0,-,0,0
+z,-,-,+,+,-,-,-,+,+,-,-,+,+,-,-,-,-,-,-,-,0,-,0,0
+z̪,-,-,+,+,-,-,-,+,+,-,-,+,+,+,-,-,-,-,-,-,0,-,0,0
+ð,-,-,+,+,-,-,-,0,+,-,-,+,+,+,-,-,-,-,-,-,0,-,0,0
+ɸ,-,-,+,+,-,-,-,-,-,-,-,+,-,0,+,-,-,-,-,-,0,-,0,0
+β,-,-,+,+,-,-,-,-,+,-,-,+,-,0,+,-,-,-,-,-,0,-,0,0
+θ,-,-,+,+,-,-,-,0,-,-,-,+,+,+,-,-,-,-,-,-,0,-,0,0
+ɧ,-,-,+,+,+,-,-,0,-,-,-,-,+,+,-,+,-,0,-,-,0,-,0,0
+ɕ,-,-,+,+,+,-,-,0,-,-,-,-,+,+,-,+,-,-,-,-,0,-,0,0
+ɬ,-,-,+,+,+,+,-,0,-,-,-,+,+,-,-,0,0,0,-,-,0,-,0,0
+ɬ̪,-,-,+,+,+,+,-,0,-,-,-,+,+,+,-,0,0,0,-,-,0,-,0,0
+ɮ,-,-,+,+,+,+,-,0,+,-,-,-,+,-,-,0,0,0,-,-,0,-,0,0
+ʑ,-,-,+,+,+,-,-,0,+,-,-,-,+,+,-,+,-,-,-,-,0,-,0,0
+ɱ,-,+,+,-,0,-,+,0,+,-,-,+,-,0,+,0,0,0,-,-,0,-,0,0
+ʔ,-,+,-,-,-,-,-,0,-,-,+,-,-,0,-,-,-,-,-,-,0,-,0,0
+ŋ,-,+,+,-,-,-,+,0,+,-,-,-,-,0,-,+,-,+,-,-,0,-,0,0
+ɳ,-,+,+,-,-,-,+,0,+,-,-,-,+,0,-,-,-,-,-,-,0,-,0,0
+ɴ,-,+,+,-,-,-,+,0,+,-,-,-,-,0,-,-,-,+,-,-,0,-,0,0
+m,-,+,+,-,-,-,+,0,+,-,-,+,-,0,+,-,-,-,-,-,0,-,0,0
+n,-,+,+,-,-,-,+,0,+,-,-,+,+,-,-,-,-,-,-,-,0,-,0,0
+n̪,-,+,+,-,-,-,+,0,+,-,-,+,+,+,-,-,-,-,-,-,0,-,0,0
+ɲ,-,+,+,-,-,-,+,0,+,-,-,-,-,0,-,+,-,-,-,-,0,-,0,0
+ɥ,-,+,-,+,0,-,-,0,+,-,-,-,-,0,+,+,-,-,+,-,+,-,0,0
+ɰ,-,+,-,+,0,-,-,0,+,-,-,-,-,0,-,+,-,0,-,-,+,-,0,0
+ʋ,-,+,-,+,0,-,-,0,+,-,-,+,-,0,+,0,0,0,-,-,0,-,0,0
+ʀ,-,+,+,+,0,-,-,0,+,-,-,-,-,0,-,-,-,+,-,-,0,-,0,0
+ʙ,-,+,+,+,0,-,-,0,+,-,-,+,-,0,+,0,0,0,-,-,0,-,0,0
+ʟ,-,+,+,+,0,+,-,0,+,-,-,-,-,0,-,+,-,0,-,-,0,-,0,0
+ɭ,-,+,+,+,0,+,-,0,+,-,-,-,+,-,-,0,0,0,-,-,0,-,0,0
+ɽ,-,+,+,+,0,-,-,0,+,-,-,-,+,-,-,0,0,0,-,-,0,-,0,0
+ʎ,-,+,+,+,0,+,-,0,+,-,-,-,+,+,-,+,-,-,-,-,0,-,0,0
+r,-,+,+,+,0,-,-,0,+,-,-,+,+,-,-,0,0,0,-,-,0,-,0,0
+r̪,-,+,+,+,0,-,-,0,+,-,-,+,+,+,-,0,0,0,-,-,0,-,0,0
+ɫ,-,+,+,+,0,+,-,0,+,-,-,+,+,-,-,-,-,+,-,-,0,-,0,0
+ɺ,-,+,+,+,0,+,-,0,+,-,-,+,+,-,-,0,0,0,-,-,0,-,0,0
+ɾ,-,+,+,+,0,-,-,0,+,-,-,+,+,-,-,0,0,0,-,-,0,-,0,0
+ʍ,-,+,-,+,-,-,-,0,-,-,-,-,-,0,+,+,-,+,+,-,0,-,0,0
+h,-,+,+,+,-,-,-,0,-,-,-,-,-,0,-,-,-,-,-,-,0,-,0,0
+j,-,+,-,+,-,-,-,0,+,-,-,-,-,0,-,+,-,-,-,-,0,-,0,0
+w,-,+,-,+,-,-,-,0,+,-,-,-,-,0,+,+,-,+,+,-,0,-,0,0
+ɹ,-,+,-,+,-,-,-,0,+,-,-,+,+,-,-,+,-,+,+,-,0,-,0,0
+ɻ,-,+,-,+,-,-,-,0,+,-,-,-,+,-,-,-,-,-,-,-,0,-,0,0
+l,-,+,+,+,-,+,-,0,+,-,-,+,+,-,-,-,-,-,-,-,0,-,0,0
+l̪,-,+,+,+,-,+,-,0,+,-,-,+,+,+,-,-,-,-,-,-,0,-,0,0
+ɦ,-,+,+,+,-,-,-,0,+,-,-,-,-,0,-,-,-,-,-,-,0,-,0,0
+ɑ,+,+,-,+,0,-,-,0,+,-,-,0,-,0,-,-,+,+,-,-,+,-,0,0
+ɘ,+,+,-,+,0,-,-,0,+,-,-,0,-,0,-,-,-,-,-,-,+,-,0,0
+ɞ,+,+,-,+,0,-,-,0,+,-,-,0,-,0,+,-,-,-,+,-,-,-,0,0
+ɤ,+,+,-,+,0,-,-,0,+,-,-,0,-,0,-,-,-,+,-,-,+,-,0,0
+ɵ,+,+,-,+,0,-,-,0,+,-,-,0,-,0,+,-,-,-,+,-,+,-,0,0
+ʉ,+,+,-,+,0,-,-,0,+,-,-,0,-,0,+,+,-,-,+,-,+,-,0,0
+a,+,+,-,+,-,-,-,0,+,-,-,0,-,0,-,-,+,-,-,-,+,-,0,0
+e,+,+,-,+,-,-,-,0,+,-,-,0,-,0,-,-,-,-,-,-,+,-,0,0
+i,+,+,-,+,-,-,-,0,+,-,-,0,-,0,-,+,-,-,-,-,+,-,0,0
+o,+,+,-,+,-,-,-,0,+,-,-,0,-,0,-,-,-,+,+,-,+,-,0,0
+u,+,+,-,+,-,-,-,0,+,-,-,0,-,0,+,+,-,+,+,-,+,-,0,0
+y,+,+,-,+,-,-,-,0,+,-,-,0,-,0,+,+,-,-,+,-,+,-,0,0
+æ,+,+,-,+,-,-,-,0,+,-,-,0,-,0,-,-,+,-,-,-,+,-,0,0
+ø,+,+,-,+,-,-,-,0,+,-,-,0,-,0,-,-,-,-,+,-,+,-,0,0
+œ,+,+,-,+,-,-,-,0,+,-,-,0,-,0,-,-,-,-,+,-,-,-,0,0
+ɒ,+,+,-,+,-,-,-,0,+,-,-,0,-,0,-,-,+,+,+,-,+,-,0,0
+ɔ,+,+,-,+,-,-,-,0,+,-,-,0,-,0,-,-,-,+,+,-,-,-,0,0
+ə,+,+,-,+,-,-,-,0,+,-,-,0,-,0,-,-,-,+,-,-,-,-,0,0
+ɘ,+,+,-,+,-,-,-,0,+,-,-,0,-,0,-,-,-,+,-,-,-,-,0,0
+ɵ,+,+,-,+,-,-,-,0,+,-,-,0,-,0,-,-,-,+,+,-,-,-,0,0
+ɞ,+,+,-,+,-,-,-,0,+,-,-,0,-,0,-,-,-,+,+,-,+,-,0,0
+ɜ,+,+,-,+,-,-,-,0,+,-,-,0,-,0,-,-,-,+,-,-,+,-,0,0
+ɛ,+,+,-,+,-,-,-,0,+,-,-,0,-,0,-,-,-,-,-,-,-,-,0,0
+ɨ,+,+,-,+,-,-,-,0,+,-,-,0,-,0,-,+,-,+,-,-,+,-,0,0
+ɪ,+,+,-,+,-,-,-,0,+,-,-,0,-,0,-,+,-,-,-,-,-,-,0,0
+ɯ,+,+,-,+,-,-,-,0,+,-,-,0,-,0,-,+,-,+,-,-,-,-,0,0
+ɶ,+,+,-,+,-,-,-,0,+,-,-,0,-,0,-,-,+,-,+,-,+,-,0,0
+ʊ,+,+,-,+,-,-,-,0,+,-,-,0,-,0,-,+,-,+,+,-,-,-,0,0
+ɐ,+,+,-,+,-,-,-,0,+,-,-,0,-,0,-,-,-,+,-,-,+,-,0,0
+ʌ,+,+,-,+,-,-,-,0,+,-,-,0,-,0,-,-,-,+,-,-,+,-,0,0
+ʏ,+,+,-,+,-,-,-,0,+,-,-,0,-,0,-,+,-,-,+,-,-,-,0,0
+˩,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-,-
+˨,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,+,-
+˧,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+˦,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,-,+
+˥,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,+,+

panphon/data/sort_order.yml

@@ -0,0 +1,40 @@
+- name: syl
+  reverse: True
+- name: son
+  reverse: True
+- name: cont
+  reverse: True
+- name: cons
+  reverse: True
+- name: delrel
+  reverse: True
+- name: nas
+  reverse: True
+- name: lat
+  reverse: True
+- name: sg
+  reverse: True
+- name: cg
+  reverse: True
+- name: hi
+  reverse: True
+- name: lo
+  reverse: True
+- name: lab
+  reverse: True
+- name: ant
+  reverse: True
+- name: cor
+  reverse: True
+- name: distr
+  reverse: True
+- name: back
+  reverse: True
+- name: round
+  reverse: True
+- name: tense
+  reverse: True
+- name: long
+  reverse: True
+- name: voi
+  reverse: True

panphon/data/sort_order_schema.yml

@@ -0,0 +1,10 @@
+type: seq
+sequence:
+  - type: map
+    mapping:
+      "name":
+        type: str
+        required: yes
+      "reverse":
+        type: bool
+        required: yes

panphon/distance.py

@@ -0,0 +1,837 @@
+from __future__ import (absolute_import, division, print_function,
+                        unicode_literals)
+
+import os.path
+from functools import partial
+
+import editdistance
+import numpy as np
+import regex as re
+import pkg_resources
+import yaml
+
+from . import _panphon, permissive, featuretable, xsampa
+
+def zerodiviszero(f):
+    def wrapper(*args, **kwargs):
+        try:
+            return f(*args, **kwargs)
+        except ZeroDivisionError:
+            return 0
+    return wrapper
+
+
+def xsampaopt(f):
+    def wrapper(*args, **kwargs):
+        if 'xsampa' in kwargs and kwargs['xsampa']:
+            self, source, target = args
+            source = self.xs.convert(source)
+            target = self.xs.convert(target)
+            args = (self, source, target)
+        return f(*args, **kwargs)
+    return wrapper
+
+
+def ftstr2dict(ftstr):
+    fts = {}
+    for m in re.finditer(r'([-0+])(\w+)', ftstr):
+        v, k = m.groups()
+        fts[k] = {'-': -1, '0': 0, '+': 1}[v]
+    return fts
+
+
+class Distance(object):
+    """Measures of phonological distance."""
+
+    def __init__(self, feature_set='spe+', feature_model='segment'):
+        """Construct a `Distance` object
+
+        Args:
+            feature_set (str): feature set to be used by the `Distance` object
+            feature_model (str): feature parsing model to be used by the
+                                 `Distance` object
+        """
+        fm = {'strict': _panphon.FeatureTable,
+              'permissive': permissive.PermissiveFeatureTable,
+              'segment': featuretable.FeatureTable}
+        self.fm = fm[feature_model](feature_set=feature_set)
+        self.xs = xsampa.XSampa()
+        self.dolgo_prime = self._dolgopolsky_prime()
+
+    def _dolgopolsky_prime(self, filename=os.path.join('data', 'dolgopolsky_prime.yml')):
+        """Reads dolgopolsky classes and constructs function cascade
+
+        Args:
+            filename (str): path to YAML file (from panphon root) containing
+                            dolgopolsky classes
+        """
+        filename = pkg_resources.resource_filename(
+            __name__, filename)
+        with open(filename, 'r') as f:
+            rules = []
+            dolgo_prime = yaml.load(f.read(), Loader=yaml.FullLoader)
+            for rule in dolgo_prime:
+                rules.append((ftstr2dict(rule['def']), rule['label']))
+        return rules
+
+    def map_to_dolgo_prime(self, s):
+        """Map a string to dolgopolsky' classes
+
+        Args:
+            s (unicode): IPA word
+
+        Returns:
+            (unicode): word with all segments collapsed to D' classes
+        """
+        segs = []
+        for seg in self.fm.seg_regex.finditer(s):
+            fts = self.fm.fts(seg.group(0))
+            for mask, label in self.dolgo_prime:
+                if fts >= mask:
+                    segs.append(label)
+                    break
+        return ''.join(segs)
+
+    def levenshtein_distance(self, source, target):
+        """Slow implementation of Levenshtein distance using NumPy arrays
+
+        Args:
+            source (unicode): source word
+            target (unicode): target word
+
+        Returns:
+            int: minimum number of Levenshtein edits required to get from
+                 `source` to `target`
+        """
+        if len(source) < len(target):
+            return self.levenshtein_distance(target, source)
+        # So now we have len(source) >= len(target).
+        if len(target) == 0:
+            return len(source)
+        # We call tuple() to force strings to be used as sequences
+        # ('c', 'a', 't', 's') - numpy uses them as values by default.
+        source = np.array(tuple(source))
+        target = np.array(tuple(target))
+        # We use a dynamic programming algorithm, but with the
+        # added optimization that we only need the last two rows
+        # of the matrix.
+        previous_row = np.arange(target.size + 1)
+        for s in source:
+            # Insertion (target grows longer than source):
+            current_row = previous_row + 1
+            # Substitution or matching:
+            # Target and source items are aligned, and either
+            # are different (cost of 1), or are the same (cost of 0).
+            current_row[1:] = np.minimum(current_row[1:], np.add(previous_row[:-1], target != s))
+            # Deletion (target grows shorter than source):
+            current_row[1:] = np.minimum(current_row[1:], current_row[0:-1] + 1)
+            previous_row = current_row
+        return previous_row[-1]
+
+    def fast_levenshtein_distance(self, source, target):
+        """Wrapper for the distance function in the Levenshtein module
+
+        Args:
+            source (unicode): source word
+            target (unicode): target word
+
+        Returns:
+            int: minimum number of Levenshtein edits required to get from
+                 `source` to `target`
+        """
+        return int(editdistance.eval(source, target))
+
+    def fast_levenshtein_distance_div_maxlen(self, source, target):
+        """Levenshtein distance divided by maxlen
+
+        Args:
+            source (unicode): source word
+            target (unicode): target word
+
+        Returns:
+            int: minimum number of Levenshtein edits required to get from
+                 `source` to `target` divided by the length of the longest
+                 of these arguments
+        """
+        maxlen = max(len(source), len(target))
+        return int(editdistance.eval(source, target)) / maxlen
+
+    def dolgo_prime_distance(self, source, target):
+        """Levenshtein distance using D' phonetic equivalence classes
+
+        `source` and `target` are converted to dolgopolsky' equivalence classes
+        (each segment is mapped to the appropriate class) and then the
+        Levenshtein distance between the resulting representations is
+        computed.
+
+        Args:
+            source (unicode): source word
+            target (unicode): target word
+
+        Returns:
+            int: minimum number of Levenshtein edits required to get from
+                 dolgopolsky' versions of `source` to `target`
+        """
+        source = self.map_to_dolgo_prime(source)
+        target = self.map_to_dolgo_prime(target)
+        return self.fast_levenshtein_distance(source, target)
+
+    @zerodiviszero
+    @xsampaopt
+    def dolgo_prime_distance_div_maxlen(self, source, target, xsampa=False):
+        """Levenshtein distance using D' classes, normalized by max length
+
+        `source` and `target` are converted to dolgopolsky' equivalence classes
+        (each segment is mapped to the appropriate class) and then the
+        Levenshtein distance between the resulting representations is
+        computed. The result is divided by the length of the longest argument
+        (`source` or `target`) after mapping to D' classes.
+
+        Args:
+            source (unicode): source word
+            target (unicode): target word
+
+        Returns:
+            int: minimum number of Levenshtein edits required to get from
+                 dolgopolsky' versions of `source` to `target`
+        """
+        source = self.map_to_dolgo_prime(source)
+        target = self.map_to_dolgo_prime(target)
+        maxlen = max(len(source), len(target))
+        return self.fast_levenshtein_distance(source, target) / maxlen
+
+    def min_edit_distance(self, del_cost, ins_cost, sub_cost, start, source, target):
+        """Return minimum edit distance, parameterized, slow
+
+        Args:
+            del_cost (function): cost function for deletion
+            ins_cost (function): cost function for insertion
+            sub_cost (function): cost function for substitution
+            start (sequence): start symbol: string for strings, list for lists,
+                              list of list for list of lists
+            source (sequence): source string/sequence of feature vectors
+            target (sequence): target string/sequence of feature vectors
+
+        Returns:
+            Number: minimum edit distance from source to target, with edit costs
+                    as defined
+        """
+        # Get lengths of source and target
+        n, m = len(source), len(target)
+        source, target = start + source, start + target
+        # Create "matrix"
+        d = []
+        for i in range(n + 1):
+            d.append((m + 1) * [None])
+        # Initialize "matrix"
+        d[0][0] = 0
+        for i in range(1, n + 1):
+            d[i][0] = d[i - 1][0] + del_cost(source[i])
+        for j in range(1, m + 1):
+            d[0][j] = d[0][j - 1] + ins_cost(target[j])
+        # Recurrence relation
+        for i in range(1, n + 1):
+            for j in range(1, m + 1):
+                d[i][j] = min([
+                    d[i - 1][j] + del_cost(source[i]),
+                    d[i - 1][j - 1] + sub_cost(source[i], target[j]),
+                    d[i][j - 1] + ins_cost(target[j]),
+                ])
+        return d[n][m]
+
+    def feature_difference(self, ft1, ft2):
+        """Given two feature values, return the difference divided by 2 *deprecated*
+
+        Args:
+            ft1 (int): feature value in {1, 0, -1}
+            ft2 (int): feature value in {1, 0, -1}
+
+        Returns:
+            float: half the absolute value of the difference between ft1 and ft2
+        """
+        return abs(ft1 - ft2) / 2
+
+    def unweighted_deletion_cost(self, v1, gl_wt=1.0):
+        """Return cost of deleting segment corresponding to feature vector
+
+        Features are not weighted; features specified as '0' add 0.5 to the raw
+        deletion cost; other features add 1 to the raw deletion cost; the cost
+        is normalized by the number of features
+
+        Args:
+            v1 (list): vector of feature values
+            global_weight (Number): global weighting factor
+
+        Returns:
+            float: sum of feature costs divided by the number of features and
+                   multiplied by a global weighting factor
+        """
+        assert isinstance(v1, list)
+        return sum(map(lambda x: 0.5 if x == 0 else 1, v1)) / len(v1) * gl_wt
+
+    def unweighted_substitution_cost(self, v1, v2):
+        """Given two feature vectors, return the difference
+
+        Args:
+            v1 (list): vector of feature values
+            v2 (list): vector of feature values
+
+        Returns:
+            float: sum of the differences between the features in `v1` and `v2`,
+                   divided by the number of features
+        """
+        return sum([abs(ft1 - ft2) / 2 for (ft1, ft2) in zip(v1, v2)]) / len(v1)
+
+    def unweighted_insertion_cost(self, v1, gl_wt=1.0):
+        """Return cost of inserting segment corresponding to feature vector
+
+        Features are not weighted; features with the value '0' add 0.5 to the
+        raw cost; other features add 1.0 to the raw cost; the raw cost is then
+        normalized by the number of features
+
+        Args:
+            v1 (list): vector of feature values
+            global_weight (Number): global weighting factor
+
+        Returns:
+            float: sum of the costs of inserting each of the features in `v1`
+                   divided by the number of features in the vector and
+                   multiplied by a global weighting factor
+        """
+        return sum(map(lambda x: 0.5 if x == 0 else 1, v1)) / len(v1) * gl_wt
+
+    @xsampaopt
+    def feature_edit_distance(self, source, target, xsampa=False):
+        """String edit distance with equally-weighed features.
+
+        All articulatory features are given equal weight. The distance between
+        an unspecified value and a specified value is smaller than the distance
+        between two features with oppoiste values.
+
+        Args:
+            source (unicode): source string
+            target (unicode): target string
+
+        Returns:
+            float: feature edit distance with equally-weighed features an insdel
+                   costs set so insdel operations cost as much, roughly, as
+                   substituting a whole segment
+        """
+        return self.min_edit_distance(self.unweighted_deletion_cost,
+                                      self.unweighted_insertion_cost,
+                                      self.unweighted_substitution_cost,
+                                      [[]],
+                                      self.fm.word_to_vector_list(source, numeric=True),
+                                      self.fm.word_to_vector_list(target, numeric=True))
+
+    @xsampaopt
+    def jt_feature_edit_distance(self, source, target, xsampa=False):
+        """String edit distance with equally-weighed features.
+
+        All articulatory features are given equal weight. The distance between
+        an unspecified value and a specified value is smaller than the distance
+        between two features with oppoiste values. Insdel costs are cheap.
+
+        Args:
+            source (unicode): source string
+            target (unicode): target string
+            xsampa (bool): source and target are X-SAMPA
+
+        Returns:
+            float: feature edit distance with equally-weighed features and insdel
+                   costs set so insdel operations cost 1/4 as much, roughly, as
+                   substituting a whole segment
+        """
+        return self.min_edit_distance(partial(self.unweighted_deletion_cost, gl_wt=0.25),
+                                      partial(self.unweighted_insertion_cost, gl_wt=0.25),
+                                      self.unweighted_substitution_cost,
+                                      [[]],
+                                      self.fm.word_to_vector_list(source, numeric=True),
+                                      self.fm.word_to_vector_list(target, numeric=True))
+
+    @zerodiviszero
+    @xsampaopt
+    def feature_edit_distance_div_maxlen(self, source, target, xsampa=False):
+        """Like `Distance.feature_edit_distance` but normalized by maxlen
+
+        Args:
+            source (unicode): source string
+            target (unicode): target string
+            xsampa (bool): source and target are X-SAMPA
+
+        Returns:
+            float: feature edit distance with equally-weighed features and insdel
+                   costs set so insdel operations cost as much, roughly, as
+                   substituting a whole segment
+
+                   Raw result is divided by the length of the longest argument
+        """
+        source_len, target_len = len(self.fm.word_to_vector_list(source)), len(self.fm.word_to_vector_list(target))
+        maxlen = max(source_len, target_len)
+        return self.feature_edit_distance(source, target) / maxlen
+
+    @zerodiviszero
+    @xsampaopt
+    def jt_feature_edit_distance_div_maxlen(self, source, target, xsampa=False):
+        """Like `Distance.feature_edit_distance` but normalized by maxlen
+
+        Args:
+            source (unicode): source string
+            target (unicode): target string
+            xsampa (bool): source and target are X-SAMPA
+
+        Returns:
+            float: feature edit distance with equally-weighed features and insdel
+                   costs set so insdel operations cost 1/4 as much, roughly, as
+                   substituting a whole segment
+
+                   Raw result is divided by the length of the longest argument
+        """
+        source_len, target_len = len(self.fm.word_to_vector_list(source)), len(self.fm.word_to_vector_list(target))
+        maxlen = max(source_len, target_len)
+        return self.jt_feature_edit_distance(source, target) / maxlen
+
+    def phoneme_error_rate(self, hyp, ref):
+        """Phoneme error rate over lists of hypothesized and reference strings.
+        Calculates edit distance in terms of phonemes, instead of Unicode characters
+        Normalizes by the total number of phones in the reference
+
+        Args:
+            hyp (list[unicode]): hypothesized strings
+            ref (list[unicode]): reference strings
+
+        Returns:
+            float: phoneme error rate (PER)
+        """
+        if hyp and ref:
+            errors = []
+            for (h, r) in zip(hyp, ref):
+                phoneme_edits = self.min_edit_distance(
+                    lambda v: 1,
+                    lambda v: 1,
+                    lambda x,y: 0 if x == y else 1,
+                    [[]],
+                    self.fm.ipa_segs(h),
+                    self.fm.ipa_segs(r)
+                )
+                errors.append(phoneme_edits)
+            total_phones = sum([len(self.fm.ipa_segs(r)) for r in ref])
+
+            return sum(errors) / total_phones
+        else:
+            return 0.0
+
+    def feature_error_rate(self, hyp, ref, xsampa=False):
+        """Feature error rate over lists of hypothesized and reference strings.
+        
+        Args:
+            hyp (list[unicode]): hypothesized strings
+            ref (list[unicode]): reference strings
+
+        Returns:
+            float: feature error rate (FER)
+
+        """
+        if hyp and ref:
+            errors = sum([self.feature_edit_distance(h, r) for (h, r) in zip(hyp, ref)])
+            ft = featuretable.FeatureTable()
+            total_phones = sum([len(ft.ipa_segs(r)) for r in ref])
+            return errors / total_phones
+        else:
+            return 0.0
+
+    def hamming_substitution_cost(self, v1, v2):
+        """Substitution cost for feature vectors computed as Hamming distance.
+
+        Substitution cost for feature vectors computed as Hamming distance and
+        normalized by dividing this result by the length of the vectors.
+
+        Args:
+            v1 (list): feature vector
+            v2 (list): feature vector
+
+        Returns:
+            float: Hamming distance between `v1` and `v2` divided by the length
+                   of `v1` and `v2`
+        """
+        diffs = [ft1 != ft2 for (ft1, ft2) in zip(v1, v2)]
+        return sum(diffs) / len(diffs)  # Booleans are cohersed to integers.
+
+    @xsampaopt
+    def hamming_feature_edit_distance(self, source, target, xsampa=False):
+        """String edit distance with equally-weighed features.
+
+        All articulatory features are given equal weight. The distance between an
+        unspecified value and a specified value is smaller than the distance between
+        two features with oppoiste values.
+
+        The insertion and deletion cost is always one, somewhat favoring
+        substitution.
+
+        This function has no normalization but should obey the triangle
+        inequality and thus provide a true distance metric.
+
+        Args:
+            source (unicode): source string
+            target (unicode): target string
+            xsampa (bool): source and target are X-SAMPA
+
+        Returns:
+            float: Hamming feature edit distance between `source` and `target`
+                   with high insdel costs
+        """
+        return self.min_edit_distance(lambda v: 1,
+                                      lambda v: 1,
+                                      self.hamming_substitution_cost,
+                                      [[]],
+                                      self.fm.word_to_vector_list(source, numeric=True),
+                                      self.fm.word_to_vector_list(target, numeric=True))
+
+    @xsampaopt
+    def jt_hamming_feature_edit_distance(self, source, target, xsampa=False):
+        """String edit distance with equally-weighed features.
+
+        All articulatory features are given equal weight. The distance between an
+        unspecified value and a specified value is smaller than the distance between
+        two features with oppoiste values.
+
+        The insertion and deletion cost is always one, somewhat favoring
+        substitution.
+
+        This function has no normalization but should obey the triangle
+        inequality and thus provide a true distance metric.
+
+        Args:
+            source (unicode): source string
+            target (unicode): target string
+            xsampa (bool): source and target are X-SAMPA
+
+        Returns:
+            float: Hamming feature edit distance between `source` and `target`
+                   with low insdel costs (1/4 cost of total substitution)
+        """
+        return self.min_edit_distance(lambda v: 0.25,
+                                      lambda v: 0.25,
+                                      self.hamming_substitution_cost,
+                                      [[]],
+                                      self.fm.word_to_vector_list(source, numeric=True),
+                                      self.fm.word_to_vector_list(target, numeric=True))
+
+    @zerodiviszero
+    @xsampaopt
+    def hamming_feature_edit_distance_div_maxlen(self, source, target, xsampa=False):
+        """Hamming feature edit distance divded by maxlen
+
+        The same as `Distance.hamming_feature_edit_distance` except that the
+        resulting value is divided by the length of the longest argument. It
+        therefore does not obey the triangle inequality and is not a proper
+        metric.
+
+        Args:
+            source (unicode): source string
+            target (unicode): target string
+            xsampa (bool): source and target are X-SAMPA
+
+        Returns:
+            float: Hamming feature edit distance between `source` and `target`
+                   with high insdel costs, normalized by length of longest
+                   argument
+        """
+        source = self.fm.word_to_vector_list(source, numeric=True)
+        target = self.fm.word_to_vector_list(target, numeric=True)
+        maxlen = max(len(source), len(target))
+        raw = self.min_edit_distance(lambda v: 1,
+                                     lambda v: 1,
+                                     self.hamming_substitution_cost,
+                                     [[]],
+                                     source,
+                                     target)
+        return raw / maxlen
+
+    @xsampaopt
+    def jt_hamming_feature_edit_distance_div_maxlen(self, source, target, xsampa=False):
+        """Hamming feature edit distance divded by maxlen
+
+        The same as `Distance.hamming_feature_edit_distance` except that the
+        resulting value is divided by the length of the longest argument. It
+        therefore does not obey the triangle inequality and is not a proper
+        metric.
+
+        Args:
+            source (unicode): source string
+            target (unicode): target string
+            xsampa (bool): source and target are X-SAMPA
+
+        Returns:
+            float: Hamming feature edit distance between `source` and `target`
+                   with low insdel costs, normalized by length of longest
+                   argument
+        """
+        source = self.fm.word_to_vector_list(source, numeric=True)
+        target = self.fm.word_to_vector_list(target, numeric=True)
+        maxlen = max(len(source), len(target))
+        raw = self.min_edit_distance(lambda v: 0.25,
+                                     lambda v: 0.25,
+                                     self.hamming_substitution_cost,
+                                     [[]], source, target)
+        return raw / maxlen
+
+    def weighted_feature_difference(self, w, ft1, ft2):
+        """Return the weighted difference between two features *deprecated*
+
+        Args:
+            w (Number): weight
+            ft1 (str): feature value
+            ft2 (str): feature value
+
+        Returns:
+            float: difference between two features multiplied by weight; raw
+                   differences are:
+                        '+' - '-' = 1.0
+                        '-' - '+' = 1.0
+                        '+' - '0' = 0.5
+                        '-' - '0' = 0.5
+                        '0' - '+' = 0.5
+                        '0' - '-' = 0.5
+                   Raw differences are multipled by weight `w`
+        """
+        return self.feature_difference(ft1, ft2) * w
+
+    def weighted_substitution_cost(self, v1, v2, gl_wt=1.0):
+        """Given two feature vectors, return the difference
+
+        Args:
+            v1 (list): feature vector
+            v2 (list): feature vector
+
+        Returns:
+            float: sum of weighted feature difference for each feature pair in
+                   zip(v1, v2)
+        """
+        return sum([abs(ft1 - ft2) * w
+                    for (w, ft1, ft2)
+                    in zip(self.fm.weights, v1, v2)]) * gl_wt
+
+    def weighted_insertion_cost(self, v1, gl_wt=1.0):
+        """Return cost of inserting segment corresponding to feature vector
+
+        Args:
+            v1 (list): feature vector
+            gl_wt (float): global weights
+
+        Returns:
+           float: sum of weights multiplied by global weight (`gl_wt`)
+        """
+        assert isinstance(v1, list)
+        return sum(self.fm.weights) * gl_wt
+
+    def weighted_deletion_cost(self, v1, gl_wt=1.0):
+        """Return cost of deleting segment corresponding to feature vector
+
+        Args:
+            v1 (list): feature vector
+            gl_wt (float): global weights
+
+        Returns:
+           float: sum of weights multiplied by global weight (`gl_wt`)"""
+        assert isinstance(v1, list)
+        return sum(self.fm.weights) * gl_wt
+
+    def weighted_feature_edit_distance(self, source, target, xsampa=False):
+        """String edit distance with weighted features
+
+        The cost of changine an articulatory feature is weighted according to
+        the class of the feature and the subjective probability of the
+        feature changing in phonological alternation and loanword contexts.
+        These weights are stored in `Distance.weights`.
+
+        Args:
+            source (unicode): source string
+            target (uniocde): target string
+            xsampa (bool): source and target are X-SAMPA
+
+        Returns:
+            float: feature weighted string edit distance between `source` and
+                   `target`
+        """
+        return self.min_edit_distance(self.weighted_deletion_cost,
+                                      self.weighted_insertion_cost,
+                                      self.weighted_substitution_cost,
+                                      [[]],
+                                      self.fm.word_to_vector_list(source, numeric=True, xsampa=xsampa),
+                                      self.fm.word_to_vector_list(target, numeric=True, xsampa=xsampa))
+
+    @xsampaopt
+    def jt_weighted_feature_edit_distance(self, source, target, xsampa=False):
+        """String edit distance with weighted features
+
+        The cost of changine an articulatory feature is weighted according to
+        the class of the feature and the subjective probability of the
+        feature changing in phonological alternation and loanword contexts.
+        These weights are stored in `Distance.weights`.
+
+        Args:
+            source (unicode): source string
+            target (uniocde): target string
+            xsampa (bool): source and target are X-SAMPA
+
+        Returns:
+            float: feature weighted string edit distance between `source` and
+                   `target`
+        """
+        return self.min_edit_distance(partial(self.weighted_deletion_cost, gl_wt=0.25),
+                                      partial(self.weighted_insertion_cost, gl_wt=0.25),
+                                      self.weighted_substitution_cost,
+                                      [[]],
+                                      self.fm.word_to_vector_list(source, numeric=True),
+                                      self.fm.word_to_vector_list(target, numeric=True))
+
+    @zerodiviszero
+    @xsampaopt
+    def weighted_feature_edit_distance_div_maxlen(self, source, target, xsampa=False):
+        """String edit distance with weighted features, divided by maxlen
+
+        The cost of changine an articulatory feature is weighted according to
+        the class of the feature and the subjective probability of the
+        feature changing in phonological alternation and loanword contexts.
+        These weights are stored in `Distance.weights`.
+
+        Args:
+            source (unicode): source string
+            target (uniocde): target string
+            xsampa (bool): source and target are X-SAMPA
+
+        Returns:
+            float: feature weighted string edit distance between `source` and
+                   `target` divided by the length of the longest of these
+                   arguments
+        """
+        source = self.fm.word_to_vector_list(source, numeric=True, xsampa=xsampa)
+        target = self.fm.word_to_vector_list(target, numeric=True, xsampa=xsampa)
+        maxlen = max(len(source), len(target))
+        return self.min_edit_distance(self.weighted_deletion_cost,
+                                      self.weighted_insertion_cost,
+                                      self.weighted_substitution_cost,
+                                      [[]],
+                                      source,
+                                      target) / maxlen
+
+    @zerodiviszero
+    @xsampaopt
+    def jt_weighted_feature_edit_distance_div_maxlen(self, source, target, xsampa=False):
+        """String edit distance with weighted features, cheap insdel, divided by maxlen
+
+        The cost of changine an articulatory feature is weighted according to
+        the class of the feature and the subjective probability of the
+        feature changing in phonological alternation and loanword contexts.
+        These weights are stored in `Distance.weights`.
+
+        This is like `Distance.weighted_feature_edit_distance_div_maxlen` except
+        with low insdel costs (1/4 the cost of a complete substitution).
+
+        Args:
+            source (unicode): source string
+            target (uniocde): target string
+            xsampa (bool): source and target are X-SAMPA
+
+        Returns:
+            float: feature weighted string edit distance between `source` and
+                   `target` divided by the length of the longest of these
+                   arguments
+        """
+        source = self.fm.word_to_vector_list(source, numeric=True)
+        target = self.fm.word_to_vector_list(target, numeric=True)
+        maxlen = max(len(source), len(target))
+        return self.min_edit_distance(partial(self.weighted_deletion_cost, gl_wt=0.25),
+                                      partial(self.weighted_insertion_cost, gl_wt=0.25),
+                                      self.weighted_substitution_cost,
+                                      [[]],
+                                      source,
+                                      target) / maxlen
+
+    def partial_hamming_substitution_cost(self, v1, v2):
+        """Substitution cost for feature vectors computed in a manner sensitive to specification.
+
+        Substitution cost for feature vectors computed so that
+        specified-to-specified costs 1/|V| and specified-to-unspecified costs
+        1/2*|V|.
+
+        Args: v1 (list): feature vector v2 (list): feature vector
+
+        Returns: float: Special edit distance where substitutions are less
+        expensive of one of the features is not specified
+        """
+        def subcost(ft1, ft2):
+            if ft1 == ft2:
+                return 0
+            elif ft1 == 0 or ft2 == 0:
+                return 0.5
+            else:
+                return 1
+        diffs = [subcost(ft1, ft2) for (ft1, ft2) in zip(v1, v2)]
+        return sum(diffs) / len(diffs)
+
+    @xsampaopt
+    def partial_hamming_feature_edit_distance(self, source, target, xsampa=False):
+        """String edit distance with insdel cost = 1 and sub costs are 1/22 or 1/44 depending on specification.
+
+        This method implements a distance metric which is neither identical to
+        hamming distance nor to feature edit distance.
+
+        The insertion/deletion cost for segment is always 1. The cost of
+        substituting a specified feature for a specified feature is 1/|V| where
+        |V| is the number of dimensions in a feature vector. The cost of
+        substituting a feature specification for an unspecified feature is
+        1/2*|V|.
+
+        This function has no normalization and should obey the triangle
+        inequality and thus provide a true distance metric.
+
+        Args: source (unicode): source string target (unicode): target string
+            xsampa (bool): source and target are X-SAMPA
+
+        Returns: float: Partial hamming feature edit distance between `source`
+            and `target`
+        """
+        source = self.fm.word_to_vector_list(source, numeric=True)
+        target = self.fm.word_to_vector_list(target, numeric=True)
+        return self.min_edit_distance(lambda v: 1,
+                                      lambda v: 1,
+                                      self.partial_hamming_substitution_cost,
+                                      [[]],
+                                      source,
+                                      target)
+
+    @zerodiviszero
+    @xsampaopt
+    def partial_hamming_feature_edit_distance_div_maxlen(self, source, target, xsampa=False):
+        """String edit distance with insdel cost = 1 and sub costs are 1/22 or 1/44 depending on specification.
+
+        This method implements a distance metric which is neither identical to
+        hamming distance nor to feature edit distance and normalizes it by the
+        longest input.
+
+        The insertion/deletion cost for segment is always 1. The cost of
+        substituting a specified feature for a specified feature is 1/|V| where
+        |V| is the number of dimensions in a feature vector. The cost of
+        substituting a feature specification for an unspecified feature is
+        1/2*|V|.
+
+        This method is normalized and does not satisfy the triangle inequality.
+        It is thus not a true distance metric.
+
+        Args: source (unicode): source string target (unicode): target string
+            xsampa (bool): source and target are X-SAMPA
+
+        Returns: float: Normalized partial hamming feature edit distance between
+            `source` and `target`
+        """
+        source = self.fm.word_to_vector_list(source, numeric=True)
+        target = self.fm.word_to_vector_list(target, numeric=True)
+        maxlen = max(len(source), len(target))
+        return self.min_edit_distance(lambda v: 1,
+                                      lambda v: 1,
+                                      self.partial_hamming_substitution_cost,
+                                      [[]],
+                                      source,
+                                      target) / maxlen

panphon/errors.py

@@ -0,0 +1,5 @@
+# -*- coding: utf-8 -*-
+
+
+class SegmentError(Exception):
+    pass

panphon/featuretable.py

@@ -0,0 +1,590 @@
+# -*- coding: utf-8 -*-
+from __future__ import annotations
+
+from typing import Any, Pattern
+
+import os.path
+import unicodedata
+import collections
+
+import numpy
+import pkg_resources
+
+import regex as re
+import csv
+
+from . import xsampa
+from .segment import Segment
+from functools import reduce
+
+feature_sets = {
+    'spe+': (os.path.join('data', 'ipa_all.csv'),
+             os.path.join('data', 'feature_weights.csv'))
+}
+
+class SegmentSorter:
+    def __init__(self, segments,):
+        self._segments = segments
+        self._sorted=False
+
+    @property
+    def segments(self):
+        if not self._sorted:
+            self.sort_segments()
+        return self._segments
+
+    def sort_segments(self):
+        self.segments.sort(key=self.segment_key)
+
+    @staticmethod
+    def segment_key(segment_tuple):
+        segment_data=segment_tuple[1]
+        return (
+            segment_data['syl'], segment_data['son'], segment_data['cons'], segment_data['cont'],
+            segment_data['delrel'], segment_data['lat'], segment_data['nas'], segment_data['strid'],
+            segment_data['voi'], segment_data['sg'], segment_data['cg'], segment_data['ant'],
+            segment_data['cor'], segment_data['distr'], segment_data['lab'], segment_data['hi'],
+            segment_data['lo'], segment_data['back'], segment_data['round'], segment_data['velaric'],
+            segment_data['tense'], segment_data['long'], segment_data['hitone'], segment_data['hireg']
+        )
+
+
+class FeatureTable(object):
+    """The basic PanPhon object for representing the features of sets of segments.
+
+    :param feature_set str: The set of fetures to be used by the FeatureTable object.
+    """
+    TRIE_LEAF_MARKER = None
+
+    def __init__(self, feature_set: str='spe+'):
+        bases_fn, weights_fn = feature_sets[feature_set]
+        self.weights = self._read_weights(weights_fn)
+        self.segments, self.seg_dict, self.names = self._read_bases(bases_fn, self.weights)
+        self.seg_regex = self._build_seg_regex()
+        self.seg_trie = self._build_seg_trie()
+        self.longest_seg = max([len(x) for x in self.seg_dict.keys()])
+        self.xsampa = xsampa.XSampa()
+
+        self.sorted_segments = SegmentSorter(self.segments) #used for quick binary searches
+
+
+
+    @staticmethod
+    def normalize(data: str) -> str:
+        return unicodedata.normalize('NFD', data)
+
+    def _read_bases(self, fn: str, weights):
+        fn = pkg_resources.resource_filename(__name__, fn)
+        segments = []
+        with open(fn) as f:
+            reader = csv.reader(f)
+            header = next(reader)
+            names = header[1:]
+            for row in reader:
+                ipa = FeatureTable.normalize(row[0])
+                vals = [{'-': -1, '0': 0, '+': 1}[x] for x in row[1:]]
+                vec = Segment(names,
+                              {n: v for (n, v) in zip(names, vals)},
+                              weights=weights)
+                segments.append((ipa, vec))
+        seg_dict = dict(segments)
+        return segments, seg_dict, names
+
+    def _read_weights(self, weights_fn: str) -> list[float]:
+        weights_fn = pkg_resources.resource_filename(__name__, weights_fn)
+        with open(weights_fn) as f:
+            reader = csv.reader(f)
+            next(reader)
+            weights = [float(x) for x in next(reader)]
+        return weights
+
+    def _build_seg_regex(self) -> re.Pattern:
+        segs = sorted(self.seg_dict.keys(), key=lambda x: len(x), reverse=True)
+        return re.compile(r'(?P<all>{})'.format('|'.join(segs)))
+
+    def _build_seg_trie(self) -> dict:
+        trie = {}
+        for seg in self.seg_dict.keys():
+            node = trie
+            for char in seg:
+                if char not in node:
+                    node[char] = {}
+                node = node[char]
+            node[self.TRIE_LEAF_MARKER] = None
+        return trie
+
+    def fts(self, ipa: str, normalize: bool=True) -> dict[str, int]:
+        if normalize:
+            ipa = FeatureTable.normalize(ipa)
+        if ipa in self.seg_dict:
+            return self.seg_dict[ipa]
+        else:
+            return {}
+
+    def longest_one_seg_prefix(self, word: str, normalize: bool=True) -> str:
+        """Return longest Unicode IPA prefix of a word
+
+        Args:
+            word (unicode): input word as Unicode IPA string
+            normalize (bool): whether the word should be pre-normalized
+
+        Returns:
+            unicode: longest single-segment prefix of `word` in database
+        """
+        if normalize:
+            word = FeatureTable.normalize(word)
+        last_found_length = 0
+        node = self.seg_trie
+        for pos in range(len(word) + 1):
+            if pos == len(word) or word[pos] not in node:
+                return word[:last_found_length]
+            node = node[word[pos]]
+            if self.TRIE_LEAF_MARKER in node:
+                last_found_length = pos + 1
+        return ''
+
+    def ipa_segs(self, word: str, normalize: bool=True) -> list[str]:
+        """Returns a list of segments from a word
+
+        Args:
+            word (unicode): input word as Unicode IPA string
+            normalize (bool): whether to pre-normalize the word
+
+        Returns:
+            list: list of strings corresponding to segments found in `word`
+        """
+        if normalize:
+            word = FeatureTable.normalize(word)
+        return self._segs(word, include_invalid=False, normalize=normalize)
+
+    def validate_word(self, word: str, normalize: bool=True):
+        """Returns True if `word` consists exhaustively of valid IPA segments
+
+        Args:
+            word (unicode): input word as Unicode IPA string
+            normalize (bool): whether to pre-normalize the word
+
+        Returns:
+            bool: True if `word` can be divided exhaustively into IPA segments
+                  that exist in the database
+
+        """
+        return not self._segs(word, include_valid=False, include_invalid=True, normalize=normalize)
+
+    def word_fts(self, word: str, normalize: bool=True):
+        """Return a list of Segment objects corresponding to the segments in
+           word.
+
+        Args:
+            word (unicode): word consisting of IPA segments
+            normalize (bool): whether to pre-normalize the word
+
+        Returns:
+            list: list of Segment objects corresponding to word
+        """
+        return [self.fts(ipa, False) for ipa in self.ipa_segs(word, normalize)]
+
+    def word_array(self, ft_names: list[str], word: str, normalize: bool=True) -> numpy.ndarray:
+        """Return a ndarray of features namd in ft_name for the segments in word
+
+        Args:
+            ft_names (list): strings naming subset of features in self.names
+            word (unicode): word to be analyzed
+            normalize (bool): whether to pre-normalize the word
+
+        Returns:
+            ndarray: segments in rows, features in columns as [-1, 0, 1]
+        """
+        return numpy.array([s.numeric(ft_names) for s in self.word_fts(word, normalize)])
+
+    def bag_of_features(self, word: str, normalize: bool=True) -> numpy.ndarray:
+        """Return a vector in which each dimension is the number of times a feature-value pair occurs in the word
+        
+        Args:
+            word (unicode): word consisting of IPA segments
+            normalize (bool): whether to pre-normalize the word
+
+        Returns:
+            array: array of integers corresponding to a bag of feature-value pair counts
+        """
+        # we changed here !!
+        # word_features = self.word_fts(word: str, normalize: bool=True)
+        word_features = self.word_fts(word, normalize=True)
+        features = [v + f for f in self.names for v in ['+', '0', '-']]
+        bag = collections.OrderedDict()
+        for f in features:
+            bag[f] = 0
+        vdict = {-1: '-', 0: '0', 1: '+'}
+        for w in word_features:
+            for (f, v) in w.items():
+                bag[vdict[v] + f] += 1
+        return numpy.array(list(bag.values()))
+
+    def seg_known(self, segment: str, normalize: bool=True) -> bool:
+        """Return True if `segment` is in segment <=> features database
+
+        Args:
+            segment (unicode): consonant or vowel
+            normalize (bool): whether to pre-normalize the segment
+
+        Returns:
+            bool: True, if `segment` is in the database
+        """
+        if normalize:
+            segment = FeatureTable.normalize(segment)
+        return segment in self.seg_dict
+
+    def segs_safe(self, word: str, normalize: bool=True):
+        """Return a list of segments (as strings) from a word
+
+        Characters that are not valid segments are included in the list as
+        individual characters.
+
+        Args:
+            word (unicode): word as an IPA string
+            normalize (bool): whether to pre-normalize the word
+
+        Returns:
+            list: list of Unicode IPA strings corresponding to segments in
+                  `word`
+        """
+        if normalize:
+            word = FeatureTable.normalize(word)
+        return self._segs(word, include_invalid=True, normalize=normalize)
+
+    def _segs(self, word: str, *, include_valid: bool=True, include_invalid: bool, normalize: bool=True) -> list[str]:
+        if normalize:
+            word = FeatureTable.normalize(word)
+        segs = []
+        while word:
+            m = self.longest_one_seg_prefix(word, False)
+            if m:
+                if include_valid:
+                    segs.append(m)
+                word = word[len(m):]
+            else:
+                if include_invalid:
+                    segs.append(word[0])
+                word = word[1:]
+        return segs
+
+    def filter_segs(self, segs: list[str], normalize: bool=True) -> list[str]:
+        """Given list of strings, return only those which are valid segments
+
+        Args:
+            segs (list): list of IPA Unicode strings
+            normalize (bool): whether to pre-normalize the segments
+
+        Return:
+            list: list of IPA Unicode strings identical to `segs` but with
+                  invalid segments filtered out
+        """
+        return list(filter(lambda seg: self.seg_known(seg, normalize), segs))
+
+    def filter_string(self, word: str, normalize: bool=True) -> str:
+        """Return a string like the input but containing only legal IPA segments
+
+        Args:
+            word (unicode): input string to be filtered
+            normalize (bool): whether to pre-normalize the word (and return a normalized string)
+
+        Returns:
+            unicode: string identical to `word` but with invalid IPA segments
+                     absent
+
+        """
+        return ''.join(self.ipa_segs(word, normalize))
+
+    def fts_intersection(self, segs: list[str], normalize: bool=True) -> Segment:
+        """Return a Segment object containing the features shared by all segments
+
+        Args:
+            segs (list): IPA segments
+            normalize (bool): whether to pre-normalize the segments
+
+        Returns:
+            Segment: the features shared by all segments in segs
+        """
+        return reduce(lambda a, b: a & b,
+                      [self.fts(s, normalize) for s in self.filter_segs(segs, normalize)])
+
+    def fts_match_all(self, fts: dict[str, int], inv: list[str], normalize: bool=True) -> bool:
+        """Return `True` if all segments in `inv` matches the features in fts
+
+        Args:
+            fts (dict): a dictionary of features
+            inv (list): a collection of IPA segments represented as Unicode
+                        strings
+            normalize (bool): whether to pre-normalize the segments
+
+        Returns:
+            bool: `True` if all segments in `inv` match the features in `fts`
+        """
+        return all([self.fts(s, normalize) >= fts for s in inv])
+
+    def fts_match_any(self, fts: dict[str, int], inv: list[str], normalize: bool=True) -> bool:
+        """Return `True` if any segments in `inv` matches the features in fts
+
+        Args:
+            fts (dict): a dictionary of features
+            inv (list): a collection of IPA segments represented as Unicode
+                        strings
+            normalize (bool): whether to pre-normalize the segments
+
+        Returns:
+            bool: `True` if any segments in `inv` matches the features in `fts`
+        """
+        return any([self.fts(s, normalize) >= fts for s in inv])
+
+    def fts_contrast(self, fs: dict[str, int], ft_name: str, inv: list[str], normalize: bool=True) -> bool:
+        """Return `True` if there is a segment in `inv` that contrasts in feature
+        `ft_name`.
+
+        Args:
+            fs (dict): feature specifications used to filter `inv`.
+            ft_name (str): name of the feature where contrast must be present.
+            inv (list): collection of segments represented as Unicode strings.
+            normalize (bool): whether to pre-normalize the segments
+
+        Returns:
+            bool: `True` if two segments in `inv` are identical in features except
+                  for feature `ft_name`
+        """
+        inv_segs = filter(lambda x: x >= fs, map(lambda seg: self.fts(seg, normalize), inv))
+        for a in inv_segs:
+            for b in inv_segs:
+                if a != b:
+                    if a.differing_specs(b) == [ft_name]:
+                        return True
+        return False
+
+    def fts_count(self, fts: dict[str, int], inv: list[str], normalize: bool=True) -> int:
+        """Return the count of segments in an inventory matching a given
+        feature mask.
+
+        Args:
+            fts (dict): feature mask given as a set of (value, feature) tuples
+            inv (list): inventory of segments (as Unicode IPA strings)
+            normalize (bool): whether to pre-normalize the segments
+
+        Returns:
+            int: number of segments in `inv` that match feature mask `fts`
+        """
+        return len(list(filter(lambda s: self.fts(s, normalize) >= fts, inv)))
+
+    def match_pattern(self, pat: list[str], word: str, normalize: bool=True) -> list[dict[str, int]]:
+        """Implements fixed-width pattern matching.
+
+        Matches just in case pattern is the same length (in segments) as the
+        word and each of the segments in the pattern is a featural subset of the
+        corresponding segment in the word. Matches return the corresponding list
+        of feature sets; failed matches return None.
+
+        Args:
+           pat (list): pattern consisting of a sequence of feature dicts
+           word (unicode): a Unicode IPA string consisting of zero or more
+                           segments
+           normalize (bool): whether to pre-normalize the word
+
+        Returns:
+            list: corresponding list of feature dicts or, if there is no match,
+                  None
+        """
+        segs = self.word_fts(word, normalize)
+        if len(pat) != len(segs):
+            return None
+        else:
+            if all([s >= p for (s, p) in zip(segs, pat)]):
+                return segs
+
+    def match_pattern_seq(self, pat, const, normalize=True):
+        """Implements limited pattern matching. Matches just in case pattern is
+        the same length (in segments) as the constituent and each of the
+        segments in the pattern is a featural subset of the corresponding
+        segment in the word.
+
+        Args:
+            pat (list): pattern consisting of a list of feature dicts, e.g.
+                        [{'voi': 1}]
+            const (list): a sequence of Unicode IPA strings consisting of zero
+                          or more segments.
+            normalize (bool): whether to pre-normalize the segments
+
+        Returns:
+            bool: `True` if `const` matches `pat`
+        """
+        segs = [self.fts(s, normalize) for s in const]
+        if len(pat) != len(segs):
+            return False
+        else:
+            return all([s >= p for (s, p) in zip(segs, pat)])
+
+    def all_segs_matching_fts(self, ft_mask):
+        """Return segments matching a feature mask, a dict of features
+
+        Args:
+            ft_mask (list): feature mask dict, e.g. {'voi': -1, 'cont': 1}.
+
+        Returns:
+            list: segments matching `ft_mask`, sorted in reverse order by length
+        """
+        matching_segs = [ipa for (ipa, fts) in self.segments if fts >= ft_mask]
+        return sorted(matching_segs, key=lambda x: len(x), reverse=True)
+
+    def compile_regex_from_str(self, pat):
+        """Given a string describing features masks for a sequence of segments,
+        return a compiled regex matching the corresponding strings.
+
+        Args:
+            pat (str): feature masks, each enclosed in square brackets, in
+            which the features are delimited by any standard delimiter.
+
+        Returns:
+           Pattern: regular expression pattern equivalent to `pat`
+        """
+        s2n = {'-': -1, '0': 0, '+': 1}
+        seg_res = []
+        for mat in re.findall(r'\[[^]]+\]+', pat):
+            ft_mask = {k: s2n[v] for (v, k) in re.findall(r'([+-])(\w+)', mat)}
+            segs = self.all_segs_matching_fts(ft_mask)
+            seg_res.append('({})'.format('|'.join(segs)))
+        regexp = ''.join(seg_res)
+        return re.compile(regexp)
+
+    def segment_to_vector(self, seg, normalize=True):
+        """Given a Unicode IPA segment, return a list of feature specificiations
+        in canonical order.
+
+        Args:
+            seg (unicode): IPA consonant or vowel
+            normalize: whether to pre-normalize the segment
+
+        Returns:
+            list: feature specifications ('+'/'-'/'0') in the order from
+            `FeatureTable.names`
+        """
+        return self.fts(seg, normalize).strings()
+
+    def standardize_tones(self, word, nonstandard_tones=['¹','²','³','⁴','⁵']):
+        standard_tones = ['˩', '˨', '˧', '˦', '˥']
+        tone_map = dict(zip(nonstandard_tones, standard_tones))
+        standardized_word = ''.join(tone_map.get(char, char) for char in word)
+        return standardized_word
+
+
+    def word_to_vector_list(self, word, numeric=False, xsampa=False, nonstandard_tones=['¹','²','³','⁴','⁵'], normalize=True):
+        """Return a list of feature vectors, given a Unicode IPA word.
+
+        Args:
+            word (unicode): string in IPA (or X-SAMPA, provided `xsampa` is True)
+            numeric (bool): if True, return features as numeric values instead
+                            of strings
+            xsampa (bool): whether the word is in X-SAMPA instead of IPA
+            normalize: whether to pre-normalize the word (applies to IPA only)
+            nonstandard_tones (list): list of 5 nonstandard tones to be conveted
+                            to IPA tone markers.
+                            The order and numbering of the tones can be changed to reflect data.
+        Returns:
+            list: a list of lists of '+'/'-'/'0' or 1/-1/0
+        """
+        if xsampa:
+            word = self.xsampa.convert(word)
+        if nonstandard_tones:
+            word=self.standardize_tones(word,nonstandard_tones)
+        segs = self.word_fts(word, normalize or xsampa)
+
+        if numeric:
+            tensor = [x.numeric() for x in segs]
+        else:
+            tensor = [x.strings() for x in segs]
+        return tensor
+
+    def _compare_vectors(self,vector1, vector2):
+        """Compare two feature vectors digit by digit.
+
+        Args:
+            vector1 (list): First vector to compare.
+            vector2 (list): Second vector to compare.
+
+        Returns:
+            int: -1 if vector1 < vector2, 1 if vector1 > vector2, 0 if they are equal.
+        """
+        for v1, v2 in zip(vector1, vector2):
+            if v1 < v2:
+                return -1
+            elif v1 > v2:
+                return 1
+        return 0  # Vectors are equal
+
+    def _binary_search(self, segment_list, target, fuzzy_search=False):
+        """Binary search to find the segment matching the target vector.
+
+        Args:
+            segment_list (list): List of segments where each segment is a tuple (IPA, feature vector).
+            target (list): Target feature vector to search for.
+            fuzzy_search (bool): whether to search for the closest vector match if an exact match is not found.
+                If disabled and an exact match is not found, a None value is returned.
+
+        Returns:
+            str: The IPA segment matching the target vector, or None if not found.
+        """
+        low, high = 0, len(segment_list) - 1
+        best_match_index = None
+
+        while low <= high:
+            mid = (low + high) // 2
+            word_vec = self.sorted_segments.segment_key(segment_list[mid])
+            comparison = self._compare_vectors(word_vec, target)
+            if comparison == 0:
+                best_match_index = mid
+                break
+            elif comparison < 0:
+                low = mid + 1
+            else:
+                high = mid - 1
+
+        if best_match_index is None and fuzzy_search:
+            # Used for fuzzy searching
+            best_match_index = mid
+
+        if best_match_index is not None:
+            # Check neighboring rows within the range of +-5
+            best_match = segment_list[best_match_index]
+            for offset in range(-9, 5):
+                neighbor_index = best_match_index + offset
+                if 0 <= neighbor_index < len(segment_list):
+                    neighbor_segment = segment_list[neighbor_index]
+                    if not self._compare_vectors(self.sorted_segments.segment_key(neighbor_segment),target):
+                        # Check if the neighbor segment has a shorter name
+                        if len(neighbor_segment[0]) < len(best_match[0]):
+                            best_match = neighbor_segment
+            return best_match[0]
+
+        return None
+
+    def vector_list_to_word(self, tensor, xsampa=False,fuzzy_search=False):
+        """Return a Unicode IPA word, given a list of feature vectors.
+
+        Args:
+            tensor (list): a list of lists of '+'/'-'/'0' or 1/-1/0
+            xsampa (bool): whether to return the word in X-SAMPA instead of IPA
+            fuzzy_search (bool): whether to search for the closest vector match if an exact match is not found.
+                If disabled and an exact match is not found, a `ValueError` is raised.
+        Returns:
+            unicode: string in IPA (or X-SAMPA, provided `xsampa` is True)
+        """
+
+
+
+        word = ""
+        for vector in tensor:
+            match = self._binary_search(self.sorted_segments.segments, vector, fuzzy_search)
+            if match:
+                word += match
+            else:
+                raise ValueError(f"No matching segment found for vector: {vector}")
+        if xsampa:
+            word = self.xsampa.convert(word)
+
+        return word
+

panphon/permissive.py

@@ -0,0 +1,204 @@
+from __future__ import absolute_import, print_function, unicode_literals
+
+import codecs
+import copy
+import os.path
+
+import pkg_resources
+import yaml
+
+import regex as re
+import unicodecsv as csv
+
+from . import _panphon, xsampa
+
+
+def flip(s):
+    return [(b, a) for (a, b) in s]
+
+
+def update_ft_set(seg, dia):
+    seg = dict(flip(seg))
+    seg.update(dia)
+    return flip(set(seg.items()))
+
+
+class PermissiveFeatureTable(_panphon.FeatureTable):
+    """Encapsulate the segment <=> feature vector mapping implied by the files
+    data/ipa_all.csv and diacritic_definitions.yml. Uses a more permissive
+    algorithm for identifying base+diacritic combinations. To avoid a
+    combinatorial explosion, it never generates all of the base-diacritic-
+    modifier combinations, meaning it cannot easily make statements about the
+    whole set of segments."""
+
+    def __init__(self,
+                 feature_set='spe+',
+                 feature_model='strict',
+                 ipa_bases=os.path.join('data', 'ipa_bases.csv'),
+                 dias=os.path.join('data', 'diacritic_definitions.yml'),
+                 ):
+        """Construct a PermissiveFeatureTable object
+
+        Args:
+            feature_set (str): feature system (for API compatibility)
+            feature_model (str): feature parsing model (for API compatibility)
+            ipa_bases (str): path from panphon root to CSV file definining
+                             features of bases (unmodified consonants and
+                             vowels)
+            dias (str): path from panphon root to YAML file containing rules for
+                        diacritics and modifiers
+        """
+        dias = pkg_resources.resource_filename(__name__, dias)
+        self.bases, self.names = self._read_ipa_bases(ipa_bases)
+        self.prefix_dias, self.postfix_dias = self._read_dias(dias)
+        self.pre_regex, self.post_regex, self.seg_regex = self._compile_seg_regexes(self.bases, self.prefix_dias, self.postfix_dias)
+        self.xsampa = xsampa.XSampa()
+        self.weights = self._read_weights()
+
+    def _read_ipa_bases(self, fn):
+        fn = pkg_resources.resource_filename(__name__, fn)
+        with open(fn, 'rb') as f:
+            reader = csv.reader(f, encoding='utf-8', delimiter=str(','))
+            names = next(reader)[1:]
+            bases = {}
+            for row in reader:
+                seg, vals = row[0], row[1:]
+                bases[seg] = (set(zip(vals, names)))
+        return bases, names
+
+    def _read_dias(self, fn):
+        prefix, postfix = {}, {}
+        with codecs.open(fn, 'r', 'utf-8') as f:
+            defs = yaml.load(f.read(), Loader=yaml.FullLoader)
+            for dia in defs['diacritics']:
+                if dia['position'] == 'pre':
+                    prefix[dia['marker']] = dia['content']
+                else:
+                    postfix[dia['marker']] = dia['content']
+        return prefix, postfix
+
+    def _compile_seg_regexes(self, bases, prefix, postfix):
+        pre_jnd = '|'.join(prefix.keys())
+        post_jnd = '|'.join(postfix.keys())
+        bases_jnd = '|'.join(bases.keys())
+        pre_re = '({})'.format(pre_jnd)
+        post_re = '({})'.format(post_jnd)
+        seg_re = '(?P<all>(?P<pre>({})*)(?P<base>{})(?P<post>({})*))'.format(pre_jnd, bases_jnd, post_jnd)
+        return re.compile(pre_re), re.compile(post_re), re.compile(seg_re)
+
+    def _build_seg_regex(self):
+        return self.seg_regex
+
+    def _read_weights(self, filename=os.path.join('data', 'feature_weights.csv')):
+        filename = pkg_resources.resource_filename(
+            __name__, filename)
+        with open(filename, 'rb') as f:
+            reader = csv.reader(f, encoding='utf-8')
+            next(reader)
+            weights = [float(x) for x in next(reader)]
+        return weights
+
+    def fts(self, segment):
+        """Return features corresponding to segment as list of (value,
+        feature) tuples
+
+        Args:
+            segment (unicode): segment for which features are to be returned as
+                               Unicode string
+
+        Returns:
+            list: None if `segment` cannot be parsed; otherwise, a list of the
+                  features of `segment` as (value, feature) pairs
+        """
+        match = self.seg_regex.match(segment)
+        if match:
+            pre, base, post = match.group('pre'), match.group('base'), match.group('post')
+            seg = copy.deepcopy(self.bases[base])
+            for m in reversed(pre):
+                seg = update_ft_set(seg, self.prefix_dias[m])
+            for m in post:
+                seg = update_ft_set(seg, self.postfix_dias[m])
+            return set(seg)
+        else:
+            return None
+
+    def fts_match(self, fts_mask, segment):
+        """Evaluates whether a set of features 'match' a segment (are a subset
+        of that segment's features)
+
+        Args:
+            fts_mask (list): list of (value, feature) tuples
+            segment (unicode): IPA string corresponding to segment (consonant or
+                               vowel)
+        Returns:
+            bool: None if `segment` cannot be parsed; True if the feature values
+                  of `fts_mask` are a subset of those for `segment`
+        """
+        fts_seg = self.fts(segment)
+        if fts_seg:
+            fts_mask = set(fts_mask)
+            return fts_mask <= fts_seg
+        else:
+            return None
+
+    def longest_one_seg_prefix(self, word):
+        """Return longest IPA Unicode prefix of `word`
+
+        Args:
+            word (unicode): word as IPA string
+
+        Returns:
+            unicode: longest single-segment prefix of `word`
+        """
+        match = self.seg_regex.match(word)
+        if match:
+            return match.group(0)
+        else:
+            return ''
+
+    def seg_known(self, segment):
+        """Return True if the segment is valid
+
+        Args:
+            segment (unicode): a string which may or may not be a valid segment
+
+        Returns:
+            bool: True if segment can be parsed given the database of bases and
+                  diacritics
+        """
+        if self.seg_regex.match(segment):
+            return True
+        else:
+            return False
+
+    def filter_segs(self, segs):
+        """Given list of strings, return only those which are valid segments.
+
+        Args:
+            segs (list): list of unicode values
+
+        Returns:
+            list: values in `segs` that are valid segments (according to the
+                  definititions of bases and diacritics/modifiers known to the
+                  object
+        """
+        def whole_seg(seg):
+            m = self.seg_regex.match(seg)
+            if m and m.group(0) == seg:
+                return True
+            else:
+                return False
+        return list(filter(whole_seg, segs))
+
+    def segment_word_segments(self, word):
+        def n2s(s):
+            if s is None:
+                return ''
+            else:
+                return s
+        return ((n2s(m.group('pre')), n2s(m.group('base')), n2s(m.group('post')))
+                for m in self.seg_regex.finditer(word))
+
+    @property
+    def all_segs_matching_fts(self):
+        raise AttributeError("'PermissiveFeatureTable' object has no attribute 'all_segs_matching_fts'")

panphon/segment.py

@@ -0,0 +1,224 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+from collections.abc import Iterator, Iterable, Mapping
+from typing import TypeVar
+import regex as re
+
+T = TypeVar('T')
+
+# class Segment(Mapping[str, int]):
+class Segment(Mapping):
+    """Constructs a `Segment` object that models a phonological segment as a vector of features.
+    
+    :param names list[str]: An ordered list of feature names.
+    :param feature dict[str, int]: name-feature pairs for specified features.
+    :param ftstr str: A string, each /(+|0|-)\w+/ sequence of which is interpreted as a feature specification.
+    :param weights list[float]: An ordered list of feature weights/saliences.
+    """
+    def __init__(self, names: list[str], features: dict[str, int]={}, ftstr: str='', weights: "list[float]"=[]):
+        self.n2s = {-1: '-', 0: '0', 1: '+'}
+        self.s2n = {k: v for (v, k) in self.n2s.items()}
+        self.names = names
+        """Set a feature specification"""
+        self.data = {}
+        for name in names:
+            if name in features:
+                self.data[name] = features[name]
+            else:
+                self.data[name] = 0
+        for m in re.finditer(r'(\+|0|-)(\w+)', ftstr):
+            v, k = m.groups()
+            self.data[k] = self.s2n[v]
+        if weights:
+            self.weights = weights
+        else:
+            self.weights = [1 for _ in names]
+
+    def __len__(self):
+        return len(self._features)
+
+
+    def __getitem__(self, key: str) -> int:
+        """Get a feature specification"""
+        return self.data[key]
+
+    def __setitem__(self, key: str, value: int):
+        """Set a feature specification"""
+        if key in self.names:
+            self.data[key] = value
+        else:
+            raise KeyError('Unknown feature name.')
+
+    def __repr__(self) -> str:
+        """Return a string representation of a feature vector"""
+        pairs = [(self.n2s[self.data[k]], k) for k in self.names]
+        fts = ', '.join(['{}{}'.format(*pair) for pair in pairs])
+        return '<Segment [{}]>'.format(fts)
+
+    def __iter__(self) -> Iterator[str]:
+        """Return an iterator over the feature names"""
+        return iter(self.names)
+
+    def items(self) -> list[tuple[str, int]]:
+        """Return a list of the features as (name, value) pairs
+        
+        :return: List of features as (name, value) pairs
+        :rtype: list[tuple[str, int]]
+        """
+        return [(k, self.data[k]) for k in self.names]
+
+    def iteritems(self) -> Iterator[tuple[str, int]]:
+        """Return an iterator over the features as (name, value) pairs
+        
+        :return: Iterator over features as (name, value) pairs
+        :rtype: Iterator[tuple[str, int]]
+        """
+        return ((k, self.data[k]) for k in self.names)
+
+    def update(self, features: dict[str, int]):
+        """Update the objects features to match `features`.
+
+        Args:
+            features (dict): dictionary containing the new feature values
+        """
+        self.data.update(features)
+
+    def match(self, ft_mask: Segment) -> bool:
+        """Determine whether `self`'s features are a superset of `features`'s
+
+        Args:
+            features (dict): (name, value) pairs
+
+        Returns:
+           (bool): True if superset relationship holds else False
+        """
+        return all([self.data[k] == v for (k, v) in ft_mask.items()])
+
+    def __ge__(self, other: Segment) -> bool:
+        """Determine whether `self`'s features are a superset of `other`'s"""
+        return self.match(other)
+
+    def intersection(self, other: Segment) -> Segment:
+        """Return dict of features shared by `self` and `other`
+
+        Args:
+            other (Segment): object with feature specifications
+
+        Returns:
+            Segment: (name, value) pairs for each shared feature
+        """
+        data = dict(set(self.items()) & set(other.items()))
+        names = list(filter(lambda a: a in data, self.names))
+        return Segment(names, data)
+
+    def __and__(self, other: Segment) -> Segment:
+        """Return Segment of features shared by `self` and `other`"""
+        return self.intersection(other)
+
+    def numeric(self, names: list[str]=[]) -> list[int]:
+        if not names:
+            names = self.names
+        """Return feature values as a list of integers"""
+        return [self.data[k] for k in names]
+
+    def strings(self, names: list[str]=[]) -> list[str]:
+        """Return feature values as a list of strings"""
+        if not names:
+            names = self.names
+        return list(map(lambda x: self.n2s[x], self.numeric()))
+
+    def distance(self, other: Segment) -> int:
+        """Compute a distance between `self` and `other`
+
+        Args:
+            other (Segment): object to compare with `self`
+
+        Returns:
+            int: the sum of the absolute value of the difference between each
+                 of the feature values in `self` and `other`.
+        """
+        return sum(abs(a - b) for (a, b) in zip(self.numeric(), other.numeric()))
+
+    def norm_distance(self, other: Segment) -> float:
+        """Compute a distance, normalized by vector length
+
+        Args:
+            other (Segment): object to compare with `self`
+
+        Returns:
+            float: the sum of the absolute value of the difference between
+                   each of the feature values in `self` and `other`, divided
+                   by the number of features per vector.
+        """
+        return self.distance(other) / len(self.names)
+
+    def __sub__(self, other: Segment) -> float:
+        """Distance between segments, normalized by vector length"""
+        return self.norm_distance(other)
+
+    def hamming_distance(self, other) -> int:
+        """Compute Hamming distance between feature vectors
+
+        Args:
+            other (Segment): object to compare with `self`
+
+        Returns:
+            int: the unnormalized Hamming distance between the two vectors.
+        """
+        return sum(int(a != b) for (a, b) in zip(self.numeric(), other.numeric()))
+
+    def norm_hamming_distance(self, other: Segment) -> float:
+        """Compute Hamming distance, normalized by vector length
+
+        Args:
+            other (Segment): object to compare with `self`
+
+        Returns:
+            int: the normalized Hamming distance between the two vectors.
+        """
+        return self.hamming_distance(other) / len(self.names)
+
+    def weighted_distance(self, other: Segment) -> float:
+        """Compute weighted distance
+
+        Args:
+            other (Segment): object to compare with `self`
+
+        Returns:
+            float: the weighted distance between the two vectors
+        """
+        return sum([abs(a - b) * c for (a, b, c)
+                   in zip(self.numeric(), other.numeric(), self.weights)])
+
+    def norm_weighted_distance(self, other: Segment) -> float:
+        """Compute weighted distance, normalized by vector length
+
+        Args:
+            other (Segment): object to compare with `self`
+
+        Returns:
+            float: the weighted distance between the two vectors, normalized by
+                   vector length.
+        """
+        return self.weighted_distance(other) / sum(self.weights)
+
+    def specified(self) -> dict[str, int]:
+        """Return dictionary of features that are specified '+' or '-' (1 or -1)
+
+        Returns:
+            dict: each feature in `self` for which the value is not 0
+        """
+        return {k: v for (k, v) in self.data.items() if v != 0}
+
+    def differing_specs(self, other: Segment) -> list[str]:
+        """Return a list of feature names that differ in their specified values
+
+        Args:
+            other (Segment): object to compare with `self`
+
+        Returns:
+            list: the names of the features that differ in the two vectors
+        """
+        return [k for (k, v) in self.items() if other[k] != v]

panphon/sonority.py

@@ -0,0 +1,85 @@
+from __future__ import print_function, absolute_import, unicode_literals
+
+from . import _panphon
+from . import permissive
+
+from ._panphon import FeatureTable, fts
+
+
+class BoolTree(object):
+    """Simple decision tree specialized for sonority classes"""
+    def __init__(self, test=None, t_node=None, f_node=None):
+        """Construct a BoolTree object
+
+        Args:
+            test (bool): test for whether to traverse the true-node or the
+                         false-node (`BoolTree.t_node` or `BoolTree.f_node`)
+            t_node (BoolTree/Int): node to follow if test is `True`
+            f_node (BoolTree/Int): node to follow if test is `False`
+        """
+        self.test = test
+        self.t_node = t_node
+        self.f_node = f_node
+
+    def get_value(self):
+        if self.test:
+            if isinstance(self.t_node, BoolTree):
+                return self.t_node.get_value()
+            else:
+                return self.t_node
+        else:
+            if isinstance(self.f_node, BoolTree):
+                return self.f_node.get_value()
+            else:
+                return self.f_node
+
+
+class Sonority(object):
+    """Determine the sonority of a segment"""
+    def __init__(self, feature_set='spe+', feature_model='strict'):
+        """Construct a Sonority object
+
+        Args:
+            feature_set (str): features set to be used by `FeatureTable`
+            feature_model (str): 'strict' or 'permissive' feature model
+        """
+        fm = {'strict': _panphon.FeatureTable,
+              'permissive': permissive.PermissiveFeatureTable}
+        self.fm = fm[feature_model](feature_set=feature_set)
+
+    def sonority_from_fts(self, seg):
+        """Given a segment as features, returns the sonority on a scale of 1
+           to 9.
+
+        Args:
+            seg (list): collection of (value, feature) pairs representing
+                        a segment (vowel or consonant)
+
+        Returns:
+           int: sonority of `seg` between 1 and 9
+        """
+
+        def match(m):
+            return self.fm.match(fts(m), seg)
+
+        minusHi = BoolTree(match('-hi'), 9, 8)
+        minusNas = BoolTree(match('-nas'), 6, 5)
+        plusVoi1 = BoolTree(match('+voi'), 4, 3)
+        plusVoi2 = BoolTree(match('+voi'), 2, 1)
+        plusCont = BoolTree(match('+cont'), plusVoi1, plusVoi2)
+        plusSon = BoolTree(match('+son'), minusNas, plusCont)
+        minusCons = BoolTree(match('-cons'), 7, plusSon)
+        plusSyl = BoolTree(match('+syl'), minusHi, minusCons)
+        return plusSyl.get_value()
+
+    def sonority(self, seg):
+        """Given a segment as a Unicode IPA string, returns the sonority on
+           a scale of 1 to 9.
+
+        Args:
+            seg (unicode): IPA consonant or vowel
+
+        Returns:
+           int: sonority of `seg` between 1 and 9
+        """
+        return self.sonority_from_fts(self.fm.fts(seg))

panphon/xsampa.py

@@ -0,0 +1,35 @@
+from __future__ import absolute_import, print_function, unicode_literals
+
+import regex as re
+import unicodecsv as csv
+import os.path
+import pkg_resources
+
+
+class XSampa(object):
+    def __init__(self, delimiter=' '):
+        self.delimiter = delimiter
+        self.xs_regex, self.xs2ipa = self.read_xsampa_table()
+
+    def read_xsampa_table(self):
+        filename = os.path.join('data', 'ipa-xsampa.csv')
+        filename = pkg_resources.resource_filename(__name__, filename)
+        with open(filename, 'rb') as f:
+            xs2ipa = {x[1]: x[0] for x in csv.reader(f, encoding='utf-8')}
+        xs = sorted(xs2ipa.keys(), key=len, reverse=True)
+        xs_regex = re.compile('|'.join(list(map(re.escape, xs))))
+        return xs_regex, xs2ipa
+
+    def convert(self, xsampa):
+        def seg2ipa(seg):
+            ipa = []
+            while seg:
+                match = self.xs_regex.match(seg)
+                if match:
+                    ipa.append(self.xs2ipa[match.group(0)])
+                    seg = seg[len(match.group(0)):]
+                else:
+                    seg = seg[1:]
+            return ''.join(ipa)
+        ipasegs = list(map(seg2ipa, xsampa.split(self.delimiter)))
+        return ''.join(ipasegs)