Update app.py

app.py

@@ -197,9 +197,11 @@ def create_collection_name(filepath):
     print('Collection name: ', collection_name)
     return collection_name
 
-def build_qa_chain(collection_name, vector_db, file: str):
+#collection_name, vector_db, btn, llm_btn, slider_temperature, slider_maxtokens, slider_topk]
+def build_qa_chain(collection_name, vector_db, file: str, llm_option, llm_temperature, max_tokens, top_k, progress=gr.Progress()):
     print("in build_qa_chain="+file.name) 
     documents, file_name = process_file2(file)
+    
     # Load embeddings model
     #embeddings = OpenAIEmbeddings(openai_api_key=self.OPENAI_API_KEY)
 
@@ -275,6 +277,73 @@ def build_qa_chain(collection_name, vector_db, file: str):
     #vincent for new LLM
     #llm_model = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"
     #llm_model = "meta-llama/Llama-2-7b-chat-hf"
+
+    llm_model = list_llm[llm_option]
+    task="text-generation"  # Explicitly specify task
+
+    if llm_model == "mistralai/Mixtral-8x7B-Instruct-v0.1":
+        llm = HuggingFaceEndpoint(
+            repo_id=llm_model, 
+            task=task,  # Explicitly specify task
+            # model_kwargs={"temperature": temperature, "max_new_tokens": max_tokens, "top_k": top_k, "load_in_8bit": True}
+            temperature = temperature,
+            max_new_tokens = max_tokens,
+            top_k = top_k,
+            load_in_8bit = True,
+        )
+    elif llm_model in ["HuggingFaceH4/zephyr-7b-gemma-v0.1","mosaicml/mpt-7b-instruct"]:
+        raise gr.Error("LLM model is too large to be loaded automatically on free inference endpoint")
+        llm = HuggingFaceEndpoint(
+            repo_id=llm_model, 
+            task=task,  # Explicitly specify task
+            temperature = temperature,
+            max_new_tokens = max_tokens,
+            top_k = top_k,
+        )
+    elif llm_model == "microsoft/phi-2":
+        # raise gr.Error("phi-2 model requires 'trust_remote_code=True', currently not supported by langchain HuggingFaceHub...")
+        llm = HuggingFaceEndpoint(
+            repo_id=llm_model, 
+            task=task,  # Explicitly specify task
+            # model_kwargs={"temperature": temperature, "max_new_tokens": max_tokens, "top_k": top_k, "trust_remote_code": True, "torch_dtype": "auto"}
+            temperature = temperature,
+            max_new_tokens = max_tokens,
+            top_k = top_k,
+            trust_remote_code = True,
+            torch_dtype = "auto",
+        )
+    elif llm_model == "TinyLlama/TinyLlama-1.1B-Chat-v1.0":
+        llm = HuggingFaceEndpoint(
+            repo_id=llm_model, 
+            task=task,  # Explicitly specify task
+            # model_kwargs={"temperature": temperature, "max_new_tokens": 250, "top_k": top_k}
+            temperature = temperature,
+            max_new_tokens = 250,
+            top_k = top_k,
+        )
+    elif llm_model == "meta-llama/Llama-2-7b-chat-hf":
+        raise gr.Error("Llama-2-7b-chat-hf model requires a Pro subscription...")
+        llm = HuggingFaceEndpoint(
+            repo_id=llm_model,
+            task=task,  # Explicitly specify task
+            # model_kwargs={"temperature": temperature, "max_new_tokens": max_tokens, "top_k": top_k}
+            temperature = temperature,
+            max_new_tokens = max_tokens,
+            top_k = top_k,
+        )
+    else:
+        llm = HuggingFaceEndpoint(
+            repo_id=llm_model, 
+            task=task,  # Explicitly specify task
+            # model_kwargs={"temperature": temperature, "max_new_tokens": max_tokens, "top_k": top_k, "trust_remote_code": True, "torch_dtype": "auto"}
+            # model_kwargs={"temperature": temperature, "max_new_tokens": max_tokens, "top_k": top_k}
+            temperature = temperature,
+            max_new_tokens = max_tokens,
+            top_k = top_k,
+        )
+
+
+    """
     llm = HuggingFaceEndpoint(
             repo_id=llm_model, 
             task="text-generation",  # Explicitly specify task
@@ -283,7 +352,7 @@ def build_qa_chain(collection_name, vector_db, file: str):
             max_new_tokens = 250,
             top_k = 3,
         )
-    
+    """
     chain = ConversationalRetrievalChain.from_llm(
         llm,
         retriever=vector_db.as_retriever(),
@@ -320,7 +389,8 @@ def get_response(collection_name, vector_db, qa_chain, history, query, file):
 
     #vincent added 20250211
     if app.count == 0:
-        collection_name, vector_db, qa_chain = build_qa_chain(collection_name, vector_db, file)    
+        #collection_name, vector_db, qa_chain = build_qa_chain(collection_name, vector_db, file)    
+        raise gr.Error("Please initialize the Chain first!")
     result = qa_chain.invoke(
         {"question": query, "chat_history": chat_history_tuples}, return_only_outputs=True
         #{"question": query, "chat_history": format_chat_history(query, history)}, return_only_outputs=True