update tokenizer

tools.py

@@ -27,7 +27,7 @@ from PIL import Image
 from bs4 import BeautifulSoup
 from duckduckgo_search import DDGS
 from sentence_transformers import SentenceTransformer
-from transformers import BlipProcessor, BlipForQuestionAnswering, pipeline
+from transformers import BlipProcessor, BlipForQuestionAnswering, pipeline, AutoTokenizer
 
 # LangChain Ecosystem
 from langchain.docstore.document import Document
@@ -71,9 +71,18 @@ def create_llm_pipeline():
     #model_id = "mistralai/Mistral-Small-24B-Base-2501"
     model_id = "mistralai/Mistral-7B-Instruct-v0.3"
     #model_id = "Qwen/Qwen2-7B-Instruct"
+
+        # Load tokenizer explicitly with fast version
+    tokenizer = AutoTokenizer.from_pretrained(
+        model_id,
+        use_fast=True,  # Force fast tokenizer
+        add_prefix_space=True  # Only if actually needed
+    )
+
     return pipeline(
         "text-generation",
         model=model_id,
+        tokenizer = tokenizer, 
         device_map="auto",
         torch_dtype=torch.float16,
         max_new_tokens=1024,