Update app.py

added : import os
import uuid
from langchain_huggingface import HuggingFaceEndpoint

app.py

@@ -5,6 +5,10 @@
 # from ibm_watsonx_ai import Credentials
 # from langchain_ibm import WatsonxLLM, WatsonxEmbeddings
 
+import os
+import uuid
+from langchain_huggingface import HuggingFaceEndpoint
+
 from langchain_text_splitters import RecursiveCharacterTextSplitter
 from langchain_community.vectorstores import Chroma
 from langchain_community.document_loaders import PyPDFLoader
@@ -38,13 +42,16 @@ warnings.filterwarnings('ignore')
 #     return watsonx_llm
 
 def get_llm():
-    def _llm(prompt: str) -> str:
-        return client.text_generation(
-            prompt,
-            max_new_tokens=256,
-            temperature=0.5
-        )
-    return _llm
+    # This uses Hugging Face's free Inference API to run the model
+    # You don't need Ollama for this!
+    llm = HuggingFaceEndpoint(
+        repo_id="ibm-granite/granite-3.1-8b-instruct", # Use the latest available on API
+        task="text-generation",
+        max_new_tokens=512,
+        temperature=0.1,
+        huggingfacehub_api_token="YOUR_HF_TOKEN_HERE" # Get this from settings
+    )
+    return llm
 
 ## Document loader
 def document_loader(file):