Merge branch 'main' of https://huggingface.co/rbawden/modern_french_normalisation into main

config.json

@@ -9,7 +9,7 @@
   "attention_dropout": 0.0,
   "bos_token_id": 0,
   "custom_pipelines": {
-    "modern_french_normalisation": {
+    "modern-french-normalisation": {
       "default": {
         "model": {
           "pt": [
@@ -22,7 +22,8 @@
       "pt": [
         "AutoModelForSeq2SeqLM"
       ],
-      "tf": []
+      "tf": [],
+      "type": "text"
     }
   },
   "d_model": 256,
@@ -31,6 +32,7 @@
     "add_cross_attention": false,
     "architectures": null,
     "bad_words_ids": null,
+    "begin_suppress_tokens": null,
     "bos_token_id": 2,
     "chunk_size_feed_forward": 0,
     "cross_attention_hidden_size": null,
@@ -74,6 +76,7 @@
     "return_dict": true,
     "return_dict_in_generate": false,
     "sep_token_id": null,
+    "suppress_tokens": null,
     "task_specific_params": null,
     "temperature": 1.0,
     "tf_legacy_loss": false,
@@ -84,7 +87,7 @@
     "top_p": 1.0,
     "torch_dtype": null,
     "torchscript": false,
-    "transformers_version": "4.21.2",
+    "transformers_version": "4.25.1",
     "typical_p": 1.0,
     "use_bfloat16": false,
     "vocab_size": 1000

modern_french_normalisation.py

@@ -1,879 +0,0 @@
-#!/usr/bin/python
-from transformers import Pipeline, AutoModelForSeq2SeqLM, AutoTokenizer
-from transformers.tokenization_utils_base import TruncationStrategy
-from torch import Tensor
-import html.parser
-import unicodedata
-import sys, os
-import re
-import pickle
-from tqdm.auto import tqdm
-import operator
-from datasets import load_dataset
-
-
-def basic_tokenise(string):
-    # separate punctuation
-    for char in r',.;?!:)("…-':
-        string = re.sub('(?<! )' + re.escape(char) + '+', ' ' + char, string)
-    for char in '\'"’':
-        string = re.sub(char + '(?! )' , char + ' ', string)
-    return string.strip()
-
-def basic_tokenise_bs(string):
-    # separate punctuation
-    string = re.sub('(?<! )([,\.;\?!:\)\("…\'‘’”“«»\-])', r' \1', string)
-    string = re.sub('([,\.;\?!:\)\("…\'‘’”“«»\-])(?! )' , r'\1 ', string)
-    return string.strip()
-
-def homogenise(sent, allow_alter_length=False):
-    '''
-    Homogenise an input sentence by lowercasing, removing diacritics, etc.
-    If allow_alter_length is False, then only applies changes that do not alter
-    the length of the original sentence (i.e. one-to-one modifications). If True,
-    then also apply n-m replacements.
-    '''
-    sent = sent.lower()
-    # n-m replacemenets
-    if allow_alter_length:
-        for before, after in [('ã', 'an'), ('xoe', 'œ')]:
-            sent = sent.replace(before, after)
-        sent = sent.strip('-')
-    # 1-1 replacements only (must not change the number of characters
-    replace_from = "ǽǣáàâäąãăåćčçďéèêëęěğìíîĩĭıïĺľłńñňòóôõöøŕřśšşťţùúûũüǔỳýŷÿźẑżžÁÀÂÄĄÃĂÅĆČÇĎÉÈÊËĘĚĞÌÍÎĨĬİÏĹĽŁŃÑŇÒÓÔÕÖØŔŘŚŠŞŤŢÙÚÛŨÜǓỲÝŶŸŹẐŻŽſ"
-    replace_into = "ææaaaaaaaacccdeeeeeegiiiiiiilllnnnoooooorrsssttuuuuuuyyyyzzzzAAAAAAAACCCDEEEEEEGIIIIIIILLLNNNOOOOOORRSSSTTUUUUUUYYYYZZZZs"
-    table = sent.maketrans(replace_from, replace_into)
-    return sent.translate(table)
-
-######## Edit distance functions #######
-def _wedit_dist_init(len1, len2):
-    lev = []
-    for i in range(len1):
-        lev.append([0] * len2)  # initialize 2D array to zero
-    for i in range(len1):
-        lev[i][0] = i  # column 0: 0,1,2,3,4,...
-    for j in range(len2):
-        lev[0][j] = j  # row 0: 0,1,2,3,4,...
-    return lev
-
-
-def _wedit_dist_step(
-    lev, i, j, s1, s2, last_left, last_right, transpositions=False
-):
-    c1 = s1[i - 1]
-    c2 = s2[j - 1]
-
-    # skipping a character in s1
-    a = lev[i - 1][j] + _wedit_dist_deletion_cost(c1,c2)
-    # skipping a character in s2
-    b = lev[i][j - 1] + _wedit_dist_insertion_cost(c1,c2)
-    # substitution
-    c = lev[i - 1][j - 1] + (_wedit_dist_substitution_cost(c1, c2) if c1 != c2 else 0)
-
-    # pick the cheapest
-    lev[i][j] = min(a, b, c)#, d)
-
-def _wedit_dist_backtrace(lev):
-    i, j = len(lev) - 1, len(lev[0]) - 1
-    alignment = [(i, j, lev[i][j])]
-
-    while (i, j) != (0, 0):
-        directions = [
-            (i - 1, j),  # skip s1
-            (i, j - 1),  # skip s2
-            (i - 1, j - 1),  # substitution
-        ]
-
-        direction_costs = (
-            (lev[i][j] if (i >= 0 and j >= 0) else float("inf"), (i, j))
-            for i, j in directions
-        )
-        _, (i, j) = min(direction_costs, key=operator.itemgetter(0))
-
-        alignment.append((i, j, lev[i][j]))
-    return list(reversed(alignment))
-
-def _wedit_dist_substitution_cost(c1, c2):
-    if c1 == ' ' and c2 != ' ':
-        return 1000000
-    if c2 == ' ' and c1 != ' ':
-        return 30
-    for c in ",.;-!?'":
-        if c1 == c and c2 != c:
-            return 20
-        if c2 == c and c1 != c:
-            return 20
-    return 1
-
-def _wedit_dist_deletion_cost(c1, c2):
-    if c1 == ' ':
-        return 2
-    if c2 == ' ':
-        return 1000000
-    return 0.8
-
-def _wedit_dist_insertion_cost(c1, c2):
-    if c1 == ' ':
-        return 1000000
-    if c2 == ' ':
-        return 2
-    return 0.8
-
-def wedit_distance_align(s1, s2):
-    """
-    Calculate the minimum Levenshtein weighted edit-distance based alignment
-    mapping between two strings. The alignment finds the mapping
-    from string s1 to s2 that minimizes the edit distance cost, where each
-    operation is weighted by a dedicated weighting function.
-    For example, mapping "rain" to "shine" would involve 2
-    substitutions, 2 matches and an insertion resulting in
-    the following mapping:
-    [(0, 0), (1, 1), (2, 2), (3, 3), (4, 4), (4, 5)]
-    NB: (0, 0) is the start state without any letters associated
-    See more: https://web.stanford.edu/class/cs124/lec/med.pdf
-    In case of multiple valid minimum-distance alignments, the
-    backtrace has the following operation precedence:
-    1. Skip s1 character
-    2. Skip s2 character
-    3. Substitute s1 and s2 characters
-    The backtrace is carried out in reverse string order.
-    This function does not support transposition.
-    :param s1, s2: The strings to be aligned
-    :type s1: str
-    :type s2: str
-    :rtype: List[Tuple(int, int)]
-    """
-    # set up a 2-D array
-    len1 = len(s1)
-    len2 = len(s2)
-    lev = _wedit_dist_init(len1 + 1, len2 + 1)
-
-    # iterate over the array
-    for i in range(len1):
-        for j in range(len2):
-            _wedit_dist_step(
-                lev,
-                i + 1,
-                j + 1,
-                s1,
-                s2,
-                0,
-                0,
-                transpositions=False,
-            )
-
-    # backtrace to find alignment
-    alignment = _wedit_dist_backtrace(lev)
-    return alignment
-
-def _last_left_t_init(sigma):
-    return {c: 0 for c in sigma}
-
-def wedit_distance(s1, s2):
-    """
-    Calculate the Levenshtein weighted edit-distance between two strings.
-    The weighted edit distance is the number of characters that need to be
-    substituted, inserted, or deleted, to transform s1 into s2, weighted 
-    by a dedicated weighting function.
-    For example, transforming "rain" to "shine" requires three steps,
-    consisting of two substitutions and one insertion:
-    "rain" -> "sain" -> "shin" -> "shine".  These operations could have
-    been done in other orders, but at least three steps are needed.
-
-    Allows specifying the cost of substitution edits (e.g., "a" -> "b"),
-    because sometimes it makes sense to assign greater penalties to
-    substitutions.
-
-    This also optionally allows transposition edits (e.g., "ab" -> "ba"),
-    though this is disabled by default.
-
-    :param s1, s2: The strings to be analysed
-    :param transpositions: Whether to allow transposition edits
-    :type s1: str
-    :type s2: str
-    :type substitution_cost: int
-    :type transpositions: bool
-    :rtype: int
-    """
-    # set up a 2-D array
-    len1 = len(s1)
-    len2 = len(s2)
-    lev = _wedit_dist_init(len1 + 1, len2 + 1)
-
-    # retrieve alphabet
-    sigma = set()
-    sigma.update(s1)
-    sigma.update(s2)
-
-    # set up table to remember positions of last seen occurrence in s1
-    last_left_t = _last_left_t_init(sigma)
-
-    # iterate over the array
-    # i and j start from 1 and not 0 to stay close to the wikipedia pseudo-code
-    # see https://en.wikipedia.org/wiki/Damerau%E2%80%93Levenshtein_distance
-    for i in range(len1):
-        last_right_buf = 0
-        for j in range(len2):
-            last_left = last_left_t[s2[j - 1]]
-            last_right = last_right_buf
-            if s1[i - 1] == s2[j - 1]:
-                last_right_buf = j
-            _wedit_dist_step(
-                lev,
-                i + 1,
-                j + 1,
-                s1,
-                s2,
-                last_left,
-                last_right,
-                transpositions=False,
-            )
-        last_left_t[s1[i - 1]] = i
-    return lev[len1-1][len2-1]
-
-def space_after(idx, sent):
-    if idx < len(sent) -1 and sent[idx + 1] == ' ':
-        return True
-    return False
-
-def space_before(idx, sent):
-    if idx > 0 and sent[idx - 1] == ' ':
-        return True
-    return False
-
-######## Normaliation pipeline #########
-class NormalisationPipeline(Pipeline):
-
-    def __init__(self, beam_size=5, batch_size=32, tokenise_func=None, cache_file=None, no_postproc_lex=False, 
-                 no_post_clean=False, **kwargs):
-        self.beam_size = beam_size
-        # classic tokeniser function (used for alignments)
-        if tokenise_func is not None:
-            self.classic_tokenise = tokenise_func
-        else:
-            self.classic_tokenise = basic_tokenise
-
-        self.no_post_clean = no_post_clean
-        self.no_postproc_lex = no_postproc_lex
-        # load lexicon
-        if no_postproc_lex:
-            self.orig_lefff_words, self.mapping_to_lefff, self.mapping_to_lefff2 = None, None, None
-        else:
-            self.orig_lefff_words, self.mapping_to_lefff, self.mapping_to_lefff2 = self.load_lexicon(cache_file=cache_file)
-        super().__init__(**kwargs)
-
-
-    def load_lexicon(self, cache_file=None):
-        orig_lefff_words = []
-        mapping_to_lefff = {}
-        mapping_to_lefff2 = {}
-        remove = set([])
-        remove2 = set([])
-
-        # load pickled version if there
-        if cache_file is not None and os.path.exists(cache_file):
-            return pickle.load(open(cache_file, 'rb'))
-        dataset = load_dataset("sagot/lefff_morpho")
-
-        for entry in set([x['form'].lower() for x in dataset['test']]):
-            orig_lefff_words.append(entry)
-            orig_lefff_words.append("-"+entry)
-            for mod_entry in set(self._create_modified_versions(entry)):
-                if mod_entry in mapping_to_lefff and mapping_to_lefff[mod_entry] != entry:
-                    remove.add(mod_entry)
-                    if mod_entry != mod_entry.upper():
-                        remove.add(mod_entry)
-                if mod_entry not in mapping_to_lefff and mod_entry != entry:
-                    mapping_to_lefff[mod_entry] = entry
-                    if mod_entry != mod_entry.upper():
-                        mapping_to_lefff2[mod_entry.upper()] = entry.upper()
-                for mod_entry2 in set(self._create_modified_versions(mod_entry)):
-                    if mod_entry2 in mapping_to_lefff2 and mapping_to_lefff2[mod_entry2] != entry:
-                        remove2.add(mod_entry2)
-                        if mod_entry2 != mod_entry2.upper():
-                            remove2.add(mod_entry2)
-                    if mod_entry2 not in mapping_to_lefff2 and mod_entry2 != entry:
-                        mapping_to_lefff2[mod_entry2] = entry
-                        if mod_entry2 != mod_entry2.upper():
-                            mapping_to_lefff2[mod_entry2.upper()] = entry.upper()
-                for mod_entry2 in set(self._create_further_modified_versions(mod_entry)):
-                    if mod_entry2 in mapping_to_lefff2 and mapping_to_lefff2[mod_entry2] != entry:
-                        remove2.add(mod_entry2)
-                        if mod_entry2 != mod_entry2.upper():
-                            remove2.add(mod_entry2)
-                    if mod_entry2 not in mapping_to_lefff2 and mod_entry2 != entry:
-                        mapping_to_lefff2[mod_entry2] = entry
-                        if mod_entry2 != mod_entry2.upper():
-                            mapping_to_lefff2[mod_entry2.upper()] = entry.upper()
-            for mod_entry2 in set(self._create_further_modified_versions(entry)):
-                if mod_entry2 in mapping_to_lefff2 and mapping_to_lefff2[mod_entry2] != entry:
-                    remove2.add(mod_entry2)
-                    if mod_entry2 != mod_entry2.upper():
-                        remove2.add(mod_entry2)
-                if mod_entry2 not in mapping_to_lefff2 and mod_entry2 != entry:
-                    mapping_to_lefff2[mod_entry2] = entry
-                    if mod_entry2 != mod_entry2.upper():
-                        mapping_to_lefff2[mod_entry2.upper()] = entry.upper()
-                    
-        for mod_entry in list(mapping_to_lefff.keys()):
-            if mod_entry != "":
-                mapping_to_lefff["-"+mod_entry] = "-"+mapping_to_lefff[mod_entry]
-        for mod_entry2 in list(mapping_to_lefff2.keys()):
-            if mod_entry2 != "":
-                mapping_to_lefff2["-"+mod_entry2] = "-"+mapping_to_lefff2[mod_entry2]
-
-        for entry in remove:
-            del mapping_to_lefff[entry]
-        for entry in remove2:
-            del mapping_to_lefff2[entry]
-
-        if cache_file is not None:
-            pickle.dump((orig_lefff_words, mapping_to_lefff, mapping_to_lefff2), open(cache_file, 'wb'))
-        return orig_lefff_words, mapping_to_lefff, mapping_to_lefff2
-
-    def _create_modified_versions(self, entry=None):
-        if entry is None:
-            return []
-        return self._remove_diacritics(entry), self._vu_vowel_to_v_vowel(entry), self._vowel_u_to_vowel_v(entry), self._consonant_v_to_consonant_u(entry), self._y_to_i(entry), self._i_to_y(entry), self._eacute_to_e_s(entry), self._final_eacute_to_e_z(entry), self._egrave_to_eacute(entry), self._vowelcircumflex_to_vowel_s(entry), self._ce_to_ee(entry)
-
-    def _create_further_modified_versions(self, entry=None):
-        if entry is None:
-            return []
-        return self._s_to_f(entry), self._ss_to_ff(entry), self._s_to_ff(entry), self._first_s_to_f(entry), self._first_s_to_ff(entry), self._last_s_to_f(entry), self._last_s_to_ff(entry), self._sit_to_st(entry), self._ee_to_ce(entry), self._z_to_s(entry)
-
-    def _remove_diacritics(self, s=None, allow_alter_length=True):
-        # 1-1 replacements only (must not change the number of characters
-        replace_from = "ǽǣáàâäąãăåćčçďéèêëęěğìíîĩĭıïĺľłńñňòóôõöøŕřśšşťţùúûũüǔỳýŷÿźẑżžÁÀÂÄĄÃĂÅĆČÇĎÉÈÊËĘĚĞÌÍÎĨĬİÏĹĽŁŃÑŇÒÓÔÕÖØŔŘŚŠŞŤŢÙÚÛŨÜǓỲÝŶŸŹẐŻŽſ"
-        replace_into = "ææaaaaaaaacccdeeeeeegiiiiiiilllnnnoooooorrsssttuuuuuuyyyyzzzzAAAAAAAACCCDEEEEEEGIIIIIIILLLNNNOOOOOORRSSSTTUUUUUUYYYYZZZZs"
-        table = s.maketrans(replace_from, replace_into)
-        s = s.translate(table)
-        # n-m replacemenets
-        if allow_alter_length:
-            for before, after in [
-                    ('œ', 'oe'),
-                    ('æ', 'ae'),
-                    ('ƣ', 'oi'),
-                    ('ij', 'ij'),
-                    ('ȣ', 'ou'),
-                    ('Œ', 'OE'),
-                    ('Æ', 'AE'),
-                    ('Ƣ', 'OI'),
-                    ('IJ', 'IJ'),
-                    ('Ȣ', 'OU')
-                    ]:
-                s = s.replace(before, after)
-            s = s.strip('-')
-        return s
-
-    def _vu_vowel_to_v_vowel(self, s=None):
-        s = re.sub('v([aeiou])' , r'vu\1', s)
-        return s
-    
-    def _vowel_u_to_vowel_v(self, s=None):
-        s = re.sub('([aeiou])u' , r'\1v', s)
-        return s
-    
-    def _consonant_v_to_consonant_u(self, s=None):
-        s = re.sub('([^aeiou])v' , r'\1u', s)
-        return s
-    
-    def _y_to_i(self, s=None):
-        s = s.replace('y', 'i')
-        return s
-
-    def _i_to_y(self, s=None):
-        s = s.replace('i', 'y')
-        return s
-
-    def _ss_to_ff(self, s=None):
-        s = s.replace('ss', 'ff')
-        return s
-
-    def _s_to_f(self, s=None):
-        s = s.replace('s', 'f')
-        return s
-
-    def _s_to_ff(self, s=None):
-        s = s.replace('s', 'ff')
-        return s
-
-    def _first_s_to_f(self, s=None):
-        s = re.sub('s' , r'f', s)
-        return s
-
-    def _last_s_to_f(self, s=None):
-        s = re.sub('^(.*)s' , r'\1f', s)
-        return s
-
-    def _first_s_to_ff(self, s=None):
-        s = re.sub('s' , r'ff', s)
-        return s
-
-    def _last_s_to_ff(self, s=None):
-        s = re.sub('^(.*)s' , r'\1ff', s)
-        return s
-
-    def _ee_to_ce(self, s=None):
-        s = s.replace('ee', 'ce')
-        return s
-
-    def _sit_to_st(self, s=None):
-        s = s.replace('sit', 'st')
-        return s
-
-    def _z_to_s(self, s=None):
-        s = s.replace('z', 's')
-        return s
-
-    def _ce_to_ee(self, s=None):
-        s = s.replace('ce', 'ee')
-        return s
-
-    def _eacute_to_e_s(self, s=None, allow_alter_length=True):
-        if allow_alter_length:
-            s = re.sub('é(.)' , r'es\1', s)
-            s = re.sub('ê(.)' , r'es\1', s)
-        return s
-
-    def _final_eacute_to_e_z(self, s=None, allow_alter_length=True):
-        if allow_alter_length:
-            s = re.sub('é$' , r'ez', s)
-            s = re.sub('ê$' , r'ez', s)
-        return s
-
-    def _egrave_to_eacute(self, s=None):
-        s = re.sub('è(.)' , r'é\1', s)
-        return s
-
-    def _vowelcircumflex_to_vowel_s(self, s=None, allow_alter_length=True):
-        if allow_alter_length:
-            for before, after in [
-                    ('â', 'as'),
-                    ('ê', 'es'),
-                    ('î', 'is'),
-                    ('ô', 'os'),
-                    ('û', 'us'),
-                    ]:
-                s = s.replace(before, after)
-        return s
-
-    def _sanitize_parameters(self, clean_up_tokenisation_spaces=None, truncation=None, **generate_kwargs):
-        preprocess_params = {}
-        if truncation is not None:
-            preprocess_params["truncation"] = truncation
-        forward_params = generate_kwargs
-        postprocess_params = {}
-        if clean_up_tokenisation_spaces is not None:
-            postprocess_params["clean_up_tokenisation_spaces"] = clean_up_tokenisation_spaces
-
-        return preprocess_params, forward_params, postprocess_params
-
-
-    def check_inputs(self, input_length: int, min_length: int, max_length: int):
-        """
-        Checks whether there might be something wrong with given input with regard to the model.
-        """
-        return True
-
-    def make_printable(self, s):
-        '''Replace non-printable characters in a string.'''
-        return s.translate(NOPRINT_TRANS_TABLE)
-
-
-    def normalise(self, line):
-        for before, after in [('[«»\“\”]', '"'), ('[‘’]', "'"), (' +', ' '), ('\"+', '"'),
-                              ("'+", "'"), ('^ *', ''), (' *$', '')]:
-            line = re.sub(before, after, line)
-        return line.strip() + ' </s>'
-    
-    def _parse_and_tokenise(self, *args, truncation):
-        prefix = ""
-        if isinstance(args[0], list):
-            if self.tokenizer.pad_token_id is None:
-                raise ValueError("Please make sure that the tokeniser has a pad_token_id when using a batch input")
-            args = ([prefix + arg for arg in args[0]],)
-            padding = True
-
-        elif isinstance(args[0], str):
-            args = (prefix + args[0],)
-            padding = False
-        else:
-            raise ValueError(
-                f" `args[0]`: {args[0]} have the wrong format. The should be either of type `str` or type `list`"
-            )
-        inputs = [self.normalise(x) for x in args]
-        inputs = self.tokenizer(inputs, padding=padding, truncation=truncation, return_tensors=self.framework)
-        toks = []
-        for tok_ids in inputs.input_ids:
-            toks.append(" ".join(self.tokenizer.convert_ids_to_tokens(tok_ids)))
-        # This is produced by tokenisers but is an invalid generate kwargs
-        if "token_type_ids" in inputs:
-            del inputs["token_type_ids"]
-        return inputs
-    
-    def preprocess(self, inputs, truncation=TruncationStrategy.DO_NOT_TRUNCATE, **kwargs):
-        inputs = self._parse_and_tokenise(inputs, truncation=truncation, **kwargs)
-        return inputs
-
-    def _forward(self, model_inputs, **generate_kwargs):
-        in_b, input_length = model_inputs["input_ids"].shape
-        generate_kwargs["min_length"] = generate_kwargs.get("min_length", self.model.config.min_length)
-        generate_kwargs["max_length"] = generate_kwargs.get("max_length", self.model.config.max_length)
-        generate_kwargs['num_beams'] = self.beam_size
-        self.check_inputs(input_length, generate_kwargs["min_length"], generate_kwargs["max_length"])
-        output_ids = self.model.generate(**model_inputs, **generate_kwargs)
-        out_b = output_ids.shape[0]
-        output_ids = output_ids.reshape(in_b, out_b // in_b, *output_ids.shape[1:])
-        return {"output_ids": output_ids}
-
-    def postprocess(self, model_outputs, clean_up_tok_spaces=False):
-        records = []
-        for output_ids in model_outputs["output_ids"][0]:
-            record = {"text": self.tokenizer.decode(output_ids, skip_special_tokens=True,
-                                                    clean_up_tokenisation_spaces=clean_up_tok_spaces).strip()}
-            records.append(record)
-        return records
-
-    def postprocess_correct_sent(self, alignment):
-        output = []
-        for i, (orig_word, pred_word, _) in enumerate(alignment):
-            if orig_word != '':
-                postproc_word = self.postprocess_correct_word(orig_word, pred_word, alignment)
-                alignment[i] = (orig_word, postproc_word, -1) # replace prediction in the alignment
-        return alignment
-
-    def postprocess_correct_word(self, orig_word, pred_word, alignment):
-        # pred_word exists in lexicon, take it
-        orig_caps = self.get_caps(orig_word)
-        if re.match("^[0-9]+$", orig_word) or re.match("^[XVUI]+$", orig_word):
-            orig_word = orig_word.replace('U', 'V')
-#            print('DEBUG:00: ', orig_word)
-            return orig_word
-        if pred_word.lower() in self.orig_lefff_words:
-            #print('pred exists')
-#            print('DEBUG:0a: ', orig_word, " => ", pred_word)
-            return self.set_caps(pred_word, *orig_caps)
-        # otherwise, if original word exists, take that
-        if orig_word.lower() in self.orig_lefff_words:
-#            print('DEBUG:0b: ', orig_word)
-            return orig_word
-
-        pred_replacement = None
-        # otherwise if pred word is in the lexicon with some changes, take that
-        if pred_word != '' and pred_word != ' ':
-            pred_replacement = self.mapping_to_lefff.get(pred_word, None)
-        if pred_replacement is not None:
-#            print('DEBUG:1: ', pred_word, " (",  pred_replacement, ", ", *orig_caps, ")")
-#            print(" => ", self.add_orig_punct(pred_word, self.set_caps(pred_replacement, *orig_caps)))
-            return self.add_orig_punct(pred_word, self.set_caps(pred_replacement, *orig_caps))
-        # otherwise if orig word is in the lexicon with some changes, take that
-        orig_replacement = self.mapping_to_lefff.get(orig_word, None)
-        if orig_replacement is not None:
-#            print('DEBUG:2: ', pred_word, " (", orig_replacement, ", ",  *orig_caps, ")")
-#            print(" => ", self.add_orig_punct(pred_word, self.set_caps(orig_replacement, *orig_caps)))
-            return self.add_orig_punct(pred_word, self.set_caps(orig_replacement, *orig_caps))
-
-        # otherwise if pred word is in the lexicon with more changes, take that
-        if pred_word != '' and pred_word != ' ':
-            pred_replacement = self.mapping_to_lefff2.get(pred_word, None)
-        if pred_replacement is not None:
-#            print('DEBUG:3: ', pred_word, " (", pred_replacement, ", ", *orig_caps, ")")
-#            print(" => ", self.add_orig_punct(pred_word, self.set_caps(pred_replacement, *orig_caps)))
-            return self.add_orig_punct(pred_word, self.set_caps(pred_replacement, *orig_caps))
-        # otherwise if orig word is in the lexicon with more changes, take that
-        orig_replacement = self.mapping_to_lefff2.get(orig_word, None)
-        if orig_replacement is not None:
-#            print('DEBUG:4: ', pred_word, " (", orig_replacement, ", ",  *orig_caps, ")")
-#            print(" => ", self.add_orig_punct(pred_word, self.set_caps(orig_replacement, *orig_caps)))
-            return self.add_orig_punct(pred_word, self.set_caps(orig_replacement, *orig_caps))
-
-        if orig_word == pred_word:
-#            print('DEBUG:0c: <', orig_word, ">")
-            return orig_word
-        if orig_word == " " and pred_word == "":
-#            print('DEBUG:0d: <', orig_word, ">")
-            return orig_word
-
-        wed = wedit_distance(pred_word,orig_word)
-        if wed > 2:
-            print("DEBUG:O",orig_word,"(P:",pred_word,":",wed,")")
-            return orig_word
-        print("DEBUG:P",self.add_orig_punct(pred_word, self.set_caps(pred_word, *orig_caps)),"(P:",pred_word,"vs. O:",orig_word,":",wed,")")
-        return self.add_orig_punct(pred_word, self.set_caps(pred_word, *orig_caps))
-
-    def get_surrounding_punct(self, word):
-        beginning_match = re.match("^(['\-]*)", word)
-        beginning, end = '', ''
-        if beginning_match:
-            beginning = beginning_match.group(1)
-        end_match = re.match("(['\-]*)$", word)
-        if end_match:
-            end = end_match.group(1)
-        return beginning, end
-
-    def add_orig_punct(self, old_word, new_word):
-        beginning, end = self.get_surrounding_punct(old_word)
-        output = ''
-        if beginning != None and not re.match("^"+re.escape(beginning), new_word):
-            output += beginning
-        if new_word != None:
-            output += new_word
-        if end != None and not re.match(re.escape(end)+"$", new_word):
-            output += end
-        return output
-    
-    def get_caps(self, word):
-        # remove any non-alphatic characters at begining or end
-        word = word.strip("-' ")
-        first, second, allcaps = False, False, False
-        if len(word) > 0 and word[0].lower() != word[0]:
-            first = True
-        if len(word) > 1 and word[1].lower() != word[1]:
-            second = True
-        if word.upper() == word and word.lower() != word:
-            allcaps = True
-        return first, second, allcaps
-
-    def set_caps(self, word, first, second, allcaps):
-        if word == None:
-            return None
-        if allcaps:
-            return word.upper()
-        elif first and second:
-            return word[0].upper() + word[1].upper() + word[2:]
-        elif first:
-            if len(word) > 1:
-                return word[0].upper() + word[1:]
-            else:
-                return word[0].upper() + word[1:]
-        elif second:
-            if len(word) > 2:
-                return word[0] + word[1].upper() + word[2:]
-            elif len(word) > 1:
-                return word[0] + word[1].upper() + word[2:]
-            else:
-                return word[0]
-        else:
-            return word
-
-    def __call__(self, input_sents, **kwargs):
-        r"""
-        Generate the output texts using texts given as inputs.
-        Args:
-            args (`List[str]`):
-                Input text for the encoder.
-            apply_postprocessing (`Bool`):
-                Apply postprocessing using the lexicon
-            generate_kwargs:
-                Additional keyword arguments to pass along to the generate method of the model (see the generate method
-                corresponding to your framework [here](./model#generative-models)).
-        Return:
-            A list or a list of list of `dict`: Each result comes as a dictionary with the following keys:
-            - **generated_text** (`str`, present when `return_text=True`) -- The generated text.
-            - **generated_token_ids** (`torch.Tensor` or `tf.Tensor`, present when `return_tensors=True`) -- The token
-              ids of the generated text.
-        """
-        result = super().__call__(input_sents, **kwargs)
-            
-        output = []
-        for i in range(len(result)):
-            #os.sys.stderr.write(str(i) + ':' + input_sents[i].strip() + '\n')
-            input_sent, pred_sent = input_sents[i].strip(), result[i][0]['text'].strip()
-            input_sent = input_sent.replace('ſ' , 's')
-            if not self.no_post_clean:
-                pred_sent = self.post_cleaning(pred_sent)
-            alignment, pred_sent_tok = self.align(input_sent, pred_sent)
-            #print(pred_sent)
-#            print("ALIGNMENT: ", alignment)
-            if not self.no_postproc_lex:
-                alignment = self.postprocess_correct_sent(alignment)
-#            print("POSTPROCESSED ALIGNMENT: ", alignment)
-            pred_sent = self.get_pred_from_alignment(alignment)
-            if not self.no_post_clean:
-                pred_sent = self.post_cleaning(pred_sent)
-            char_spans = self.get_char_idx_align(input_sent, pred_sent, alignment)
-            output.append({'text': pred_sent, 'alignment': char_spans})
-        return output
-
-    def post_cleaning(self, s):
-        s = s.replace(' ' , '')
-        s = s.replace('ſ' , 's')
-        s = s.replace('ß' , 'ss')
-        s = s.replace('&' , 'et')
-        s = re.sub('ẽ([mbp])' , r'em\1', s)
-        s = s.replace('ẽ' , 'en')
-        s = re.sub('ã([mbp])' , r'am\1', s)
-        s = s.replace('ã' , 'an')
-        s = re.sub('õ([mbp])' , r'om\1', s)
-        s = s.replace('õ' , 'on')
-        s = re.sub('ũ([mbp])' , r'um\1', s)
-        s = s.replace('ũ' , 'un')
-        return s
-
-    def align(self, sent_ref, sent_pred):
-#        print("INPUT SENT: <",sent_ref,">")
-#        print("PRED SENT: <",sent_pred,">")
-        sent_ref_tok = self.classic_tokenise(re.sub('[  ]', '  ', sent_ref))
-        sent_pred_tok = self.classic_tokenise(re.sub('[  ]', '  ', sent_pred))
-#        print("INPUT SENT TOK: <",sent_ref_tok,">")
-#        print("PRED SENT TOK: <",sent_pred_tok,">")
-        backpointers = wedit_distance_align(homogenise(sent_ref_tok), homogenise(sent_pred_tok))
-        alignment, current_word, seen1, seen2, last_weight = [], ['', ''], [], [], 0
-        for i_ref, i_pred, weight in backpointers:
-            if i_ref == 0 and i_pred == 0:
-                continue
-            # next characters are both spaces -> add current word straight away
-            if i_ref <= len(sent_ref_tok) and sent_ref_tok[i_ref-1] == ' ' \
-                and i_pred <= len(sent_pred_tok) and sent_pred_tok[i_pred-1] == ' ' \
-                and i_ref not in seen1 and i_pred not in seen2:
-
-                # if current word is empty -> insert a space on both sides
-                if current_word[0] == '' and current_word[1] == '':
-                    alignment.append((' ', ' ', weight-last_weight))
-                # else add the current word to both sides
-                else:
-                    alignment.append((current_word[0], current_word[1], weight-last_weight))
-                last_weight = weight
-                current_word = ['', '']
-                seen1.append(i_ref)
-                seen2.append(i_pred)
-            # if space in ref and dash in pred
-            elif i_ref <= len(sent_ref_tok) and sent_ref_tok[i_ref-1] == ' ' \
-                and i_pred <= len(sent_pred_tok) and sent_pred_tok[i_pred-1] == '-' \
-                and i_ref not in seen1 and i_pred not in seen2 \
-                and current_word[0] == '' and current_word[1] == '':
-                alignment.append((' ', '', weight-last_weight))
-                last_weight = weight
-                current_word = ['', '-']
-                seen1.append(i_ref)
-                seen2.append(i_pred)
-            else:
-                end_space = '' #'░'
-                # add new character to ref
-                if i_ref <= len(sent_ref_tok) and i_ref not in seen1:
-                    if i_ref > 0:
-                        current_word[0] += sent_ref_tok[i_ref-1]
-                        seen1.append(i_ref)
-                # add new character to pred
-                if i_pred <= len(sent_pred_tok) and i_pred not in seen2:
-                    if i_pred > 0:
-                        current_word[1] += sent_pred_tok[i_pred-1] if sent_pred_tok[i_pred-1] != ' ' else ' ' #'▁'
-                        end_space = '' if space_after(i_pred, sent_pred_tok) else ''# '░'
-                        seen2.append(i_pred)
-                if i_ref <= len(sent_ref_tok) and sent_ref_tok[i_ref-1] == ' ' and current_word[0].strip() != '':
-                    alignment.append((current_word[0].strip(), current_word[1].strip() + end_space, weight-last_weight))
-                    last_weight = weight
-                    current_word = ['', '']
-                # space in ref but aligned to nothing in pred (under-translation)
-                elif i_ref <= len(sent_ref_tok) and sent_ref_tok[i_ref-1] == ' ' and current_word[1].strip() == '':
-                    alignment.append((current_word[0], current_word[1], weight-last_weight))
-                    last_weight = weight
-                    current_word = ['', '']
-                    seen1.append(i_ref)
-                    seen2.append(i_pred)
-        # final word
-        alignment.append((current_word[0].strip(), current_word[1].strip(), weight-last_weight))
-        # check that both strings are entirely covered
-        recovered1 = re.sub(' +', ' ', ' '.join([x[0] for x in alignment]))
-        recovered2 = re.sub(' +', ' ', ' '.join([x[1] for x in alignment]))
-
-        assert re.sub('[  ]+', ' ', recovered1) == re.sub('[  ]+', ' ', sent_ref_tok), \
-            '\n1: *' + re.sub('[  ]+', ' ', recovered1) + "*\n1: *" + re.sub('[  ]+', ' ', sent_ref_tok) + '*'
-        assert re.sub('[░▁ ]+', '', recovered2) == re.sub('[▁ ]+', '', sent_pred_tok), \
-            '\n2: ' + re.sub('[  ]+', ' ', recovered2) + "\n2: " + re.sub('[  ]+', ' ', sent_pred_tok)
-        return alignment, sent_pred_tok
-
-    def get_pred_from_alignment(self, alignment):
-         return re.sub(' +', ' ', ''.join([x[1] if x[1] != '' else '\n' for x in alignment]).replace('\n', ''))
-    
-    def get_char_idx_align(self, sent_ref, sent_pred, alignment):
-        covered_ref, covered_pred = 0, 0
-        ref_chars = [i for i, character in enumerate(sent_ref)] + [len(sent_ref)]  #
-        pred_chars = [i for i, character in enumerate(sent_pred)] + [len(sent_pred)]# if character not in [' ']]
-        align_idx = []
-
-        for a_ref, a_pred, _ in alignment:
-            if a_ref == '' and a_pred == '':
-                covered_pred += 1
-                continue
-            a_pred = re.sub(' +', ' ', a_pred).strip()
-            span_ref = [ref_chars[covered_ref], ref_chars[covered_ref + len(a_ref)]]
-            covered_ref += len(a_ref)
-            span_pred = [pred_chars[covered_pred], pred_chars[covered_pred + len(a_pred)]]
-            covered_pred += len(a_pred)
-            align_idx.append((span_ref, span_pred))
-
-        return align_idx
-   
-def normalise_text(list_sents, batch_size=32, beam_size=5, cache_file=None, no_postproc_lex=False, no_post_clean=False):
-    tokeniser = AutoTokenizer.from_pretrained("rbawden/modern_french_normalisation")
-    model = AutoModelForSeq2SeqLM.from_pretrained("rbawden/modern_french_normalisation")
-    normalisation_pipeline = NormalisationPipeline(model=model,
-                                                   tokenizer=tokeniser,
-                                                   batch_size=batch_size,
-                                                   beam_size=beam_size,
-                                                   cache_file=cache_file,
-                                                   no_postproc_lex=no_postproc_lex,
-                                                   no_post_clean=no_post_clean)
-    normalised_outputs = normalisation_pipeline(list_sents)
-    return normalised_outputs
-
-def normalise_from_stdin(batch_size=32, beam_size=5, cache_file=None, no_postproc_lex=False, no_post_clean=False):
-    tokeniser = AutoTokenizer.from_pretrained("rbawden/modern_french_normalisation")
-    model = AutoModelForSeq2SeqLM.from_pretrained("rbawden/modern_french_normalisation")
-    normalisation_pipeline = NormalisationPipeline(model=model,
-                                                   tokenizer=tokeniser,
-                                                   batch_size=batch_size,
-                                                   beam_size=beam_size,
-                                                   cache_file=cache_file,
-                                                   no_postproc_lex=no_postproc_lex,
-                                                   no_post_clean=no_post_clean
-                                                   )
-    list_sents = []
-    ex = ["7. Qu'vne force plus grande de ſi peu que l'on voudra, que celle auec laquelle l'eau de la hauteur de trente & vn pieds, tend à couler en bas, ſuffit pour faire admettre ce vuide apparent, & meſme ſi grãd que l'on voudra, c'eſt à dire, pour faire des-vnir les corps d'vn ſi grand interualle que l'on voudra, pourueu qu'il n'y ait point d'autre obſtacle à leur ſeparation ny à leur eſloignement, que l'horreur que la Nature a pour ce vuide apparent."]
-    for sent in sys.stdin:
-        list_sents.append(sent.strip())
-    normalised_outputs = normalisation_pipeline(list_sents)
-    for s, sent in enumerate(normalised_outputs):
-        alignment=sent['alignment']
-
-        print(sent['text'])
-        # checking that the alignment makes sense
-        #for b, a in alignment:
-        #    print('input: ' + ''.join([list_sents[s][x] for x in range(b[0], max(len(b), b[1]))]) + '')
-        #    print('pred: ' + ''.join([sent['text'][x] for x in range(a[0], max(len(a), a[1]))]) + '')
-
-    return normalised_outputs
-
-    
-if __name__ == '__main__':
-    import argparse
-    parser = argparse.ArgumentParser()
-    parser.add_argument('-k', '--batch_size', type=int, default=32, help='Set the batch size for decoding')
-    parser.add_argument('-b', '--beam_size', type=int, default=5, help='Set the beam size for decoding')
-    parser.add_argument('-i', '--input_file', type=str, default=None, help='Input file. If None, read from STDIN')
-    parser.add_argument('-c', '--cache_lexicon', type=str, default=None, help='Path to cache the lexicon file to speed up loading')
-    parser.add_argument('-n', '--no_postproc_lex', default=False, action='store_true', help='Deactivate postprocessing to speed up normalisation, but this may degrade the output')
-    parser.add_argument('-m', '--no_post_clean', default=False, action='store_true', help='Deactivate postprocessing to speed up normalisation, but this may degrade the output')
-        
-    args = parser.parse_args()
-
-    if args.input_file is None:
-         normalise_from_stdin(batch_size=args.batch_size,
-                              beam_size=args.beam_size,
-                              cache_file=args.cache_lexicon,
-                              no_postproc_lex=args.no_postproc_lex,
-                              no_post_clean=args.no_post_clean)
-    else:
-         list_sents = []
-         with open(args.input_file) as fp:
-              for line in fp:
-                   list_sents.append(line.strip())
-         output_sents = normalise_text(list_sents,
-                                       batch_size=args.batch_size,
-                                       beam_size=args.beam_size,
-                                       cache_file=args.cache_lexicon,
-                                       no_postproc_lex=args.no_postproc_lex,
-                                       no_post_clean=args.no_post_clean)
-         for output_sent in output_sents:
-              print(output_sent['text'])

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:777e3ebf4be88372d6fa982cdff430b06d61461574236c7a213a37d70bd47085
-size 25265973
+oid sha256:16e283cd4628cac0f2b36f2e8181ae0ba0f65b0e03866fde6067a0cd8e3c78d8
+size 25264557