Update app.py

app.py

@@ -6,12 +6,13 @@ import gradio as gr
 import os
 from langchain_huggingface import HuggingFaceEndpoint, ChatHuggingFace
 
-from huggingface_hub import InferenceClient
+
 from langchain_text_splitters import RecursiveCharacterTextSplitter
 from langchain_community.vectorstores import Chroma
 from langchain_community.document_loaders import PyPDFLoader
 from langchain.chains import RetrievalQA
 from langchain_community.embeddings import HuggingFaceEmbeddings
+from langchain_huggingface import HuggingFaceEndpoint, ChatHuggingFace
 
 
 # -------------------------
@@ -19,13 +20,11 @@ from langchain_community.embeddings import HuggingFaceEmbeddings
 # -------------------------
 
 # You can change this to any instruct model you like
-HF_MODEL_ID = "ibm-granite/granite-3.1-8b-instruct"
+# HF_MODEL_ID = "ibm-granite/granite-3.1-8b-instruct"
 
 # Option B: Llama 3.3 (Current industry standard)
 # HF_MODEL_ID = "meta-llama/Llama-3.3-70B-Instruct"
 
-# If you use a private token, set it as HF_TOKEN env var in Spaces
-client = InferenceClient(HF_MODEL_ID, token=os.getenv("HF_TOKEN"))
 
 
 # def get_llm():
@@ -68,13 +67,14 @@ llm_endpoint = HuggingFaceEndpoint(
     max_new_tokens=512,
     temperature=0.1,
     huggingfacehub_api_token=os.getenv("HF_TOKEN"),
-    provider="hf-inference",
+    provider="hf-inference",   # forces router.huggingface.co
     model_kwargs={"wait_for_model": True},
 )
 
 llm = ChatHuggingFace(llm=llm_endpoint)
 
 
+
 # 2. ChatHuggingFace will correctly route this to the /v1/chat/completions 
 # endpoint which the Llama-3.1-8B-Instruct model supports.
 # llm = ChatHuggingFace(llm=llm_endpoint)