Bypass TokenizersBackend bug by using Nemo tokenizer for Ministral

chat_gui.py

@@ -305,9 +305,13 @@ if st.session_state.messages and st.session_state.messages[-1]["role"] == "user"
                     # Setting use_fast=False to avoid "TokenizersBackend" errors on some environments
                     processor_class = AutoTokenizer
                     
+                    # The Ministral-3 config has an invalid tokenizer class ("TokenizersBackend")
+                    # We load the exact same Tekken tokenizer from the Nemo repository to bypass this bug on HF Spaces.
+                    tokenizer_path = "mistralai/Mistral-Nemo-Instruct-2407" if is_mistral else conf["path"]
+                    
                     st.info(f"Loading {st.session_state.current_model} via Transformers...")
                     tokenizer = processor_class.from_pretrained(
-                        conf["path"], 
+                        tokenizer_path, 
                         token=hf_token, 
                         trust_remote_code=True,
                         use_fast=False