autotoken

tools.py

@@ -23,6 +23,7 @@ from transformers import BlipProcessor, BlipForQuestionAnswering, pipeline
 from io import BytesIO
 from sentence_transformers import SentenceTransformer
 from transformers import RagRetriever, RagTokenizer, RagSequenceForGeneration
+from transformers import AutoTokenizer, AutoModelWithLMHead
 
 
 import os
@@ -105,7 +106,7 @@ llm_pipe = pipeline(
 )
 
 # Initialize RAG components
-tokenizer = RagTokenizer.from_pretrained("facebook/rag-token-base", trust_remote_code=True)
+tokenizer = AutoTokenizer.from_pretrained("facebook/rag-token-base", trust_remote_code=True)
 retriever = RagRetriever.from_pretrained(
     "facebook/rag-token-base",
     index_name="exact",           # or "legacy" for legacy FAISS index