add HF token

src/streamlit_app.py

@@ -11,6 +11,11 @@ from sentence_transformers import CrossEncoder
 import pickle
 import chromadb
 from chromadb.utils import embedding_functions
+
+hf_token = os.getenv("HF_token")  # read from Space secret
+if hf_token:
+    os.environ["HUGGINGFACE_HUB_TOKEN"] = hf_token
+    
 BASE_DIR = "/tmp/rag_app"
 os.makedirs(BASE_DIR, exist_ok=True)
 # Global variables
@@ -71,15 +76,15 @@ with tab2:
         chunks = [text_data[i:i+chunk_size] for i in range(0, len(text_data), chunk_size-overlap)]
         
         if embedding_choice == "SentencePiece":
-            model = SentenceTransformer("all-MiniLM-L6-v2")
+            model = SentenceTransformer("all-MiniLM-L6-v2",use_auth_token=hf_token)
             embeddings = model.encode(chunks, batch_size=300)
         elif embedding_choice == "TF-IDF":
             vectorizer = TfidfVectorizer()
             embeddings = vectorizer.fit_transform(chunks).toarray()
         elif embedding_choice == "BERT":
             model_name = "bert-base-uncased"
-            tokenizer = AutoTokenizer.from_pretrained(model_name)
-            model = AutoModel.from_pretrained(model_name)
+            tokenizer = AutoTokenizer.from_pretrained(model_name,token=hf_token)
+            model = AutoModel.from_pretrained(model_name,token=hf_token)
             embeddings = bert_encode(model,tokenizer,chunks)
         
         if index_choice == "FAISS":
@@ -190,7 +195,7 @@ with tab3:
         #display similarity score measure used by ReRank and illustrate what number of score means more similar and its range
         st.write("Reranking using Cross-ReRank (higher score means more relevance, and lower score means less relevance). " \
         "It is relative ranking (higher score = more relevant), not the absolute magnitude.")
-        reranker = CrossEncoder("cross-encoder/ms-marco-MiniLM-L-6-v2")
+        reranker = CrossEncoder("cross-encoder/ms-marco-MiniLM-L-6-v2",token=hf_token)
         scores = [reranker.predict([(user_query, doc)])[0] for doc in retrieved_texts]
         st.subheader("Reranked scores:")
         for doc, score in zip(retrieved_texts, scores):