Adapt code for version 3.0 of nllb-deu-moo

inference.py

@@ -1,39 +1,10 @@
 from transformers import NllbTokenizer, AutoModelForSeq2SeqLM
 
 
-def create_tokenizer_with_new_lang(model_id, new_lang):
-    """
-    Add a new language token to the tokenizer vocabulary
-    (this should be done each time after its initialization)
-    """
-    tokenizer = NllbTokenizer.from_pretrained(model_id)
-    old_len = len(tokenizer) - int(new_lang in tokenizer.added_tokens_encoder)
-    tokenizer.lang_code_to_id[new_lang] = old_len-1
-    tokenizer.id_to_lang_code[old_len-1] = new_lang
-    # always move "mask" to the last position
-    tokenizer.fairseq_tokens_to_ids["<mask>"] = len(tokenizer.sp_model) + len(tokenizer.lang_code_to_id) + tokenizer.fairseq_offset
-
-    tokenizer.fairseq_tokens_to_ids.update(tokenizer.lang_code_to_id)
-    tokenizer.fairseq_ids_to_tokens = {v: k for k, v in tokenizer.fairseq_tokens_to_ids.items()}
-    if new_lang not in tokenizer._additional_special_tokens:
-        tokenizer._additional_special_tokens.append(new_lang)
-    # clear the added token encoder; otherwise a new token may end up there by mistake
-    tokenizer.added_tokens_encoder = {}
-    tokenizer.added_tokens_decoder = {}
-
-    return tokenizer
-
-
 class Translator:
     @classmethod
-    def from_pretrained(cls, path, new_lang='moo_Latn'):
-        # Does the model need adaptation or not?
-        # model, tokenizer = create_model_with_new_lang(
-        #     model_id=path,
-        #     new_lang=new_lang,
-        #     similar_lang='deu_Latn'
-        # )
-        tokenizer = create_tokenizer_with_new_lang(path, new_lang)
+    def from_pretrained(cls, path):
+        tokenizer = NllbTokenizer.from_pretrained(path)
         model = AutoModelForSeq2SeqLM.from_pretrained(path)
         return Translator(model, tokenizer)
 

requirements.txt

@@ -1,3 +1,3 @@
-transformers==4.33
+transformers>=4.46
 sentencepiece>=0.1.99
 torch>=2.1.2