Update app.py

app.py

@@ -3,7 +3,7 @@ import logging
 import gradio as gr
 from transformers import pipeline
 
-from llama_cpp_agent.providers import LlamaCppPythonProvider
+from llama_cpp_agent.providers import LLMProvider
 from llama_cpp_agent import LlamaCppAgent, MessagesFormatterType
 from llama_cpp_agent.chat_history import BasicChatHistory
 from llama_cpp_agent.chat_history.messages import Roles
@@ -18,12 +18,8 @@ from trafilatura import fetch_url, extract
 import json
 from datetime import datetime, timezone
 from typing import List
-from langchain_community.embeddings import HuggingFaceEmbeddings
 from langchain_community.llms import HuggingFaceHub
 
-llm = None
-llm_model = None
-
 huggingface_token = os.environ.get("HUGGINGFACE_TOKEN")
 
 examples = [
@@ -50,6 +46,17 @@ def get_messages_formatter_type(model_name):
     else:
         return MessagesFormatterType.CHATML
 
+class HuggingFaceHubProvider(LLMProvider):
+    def __init__(self, model):
+        self.model = model
+
+    def create_completion(self, prompt, **kwargs):
+        response = self.model(prompt)
+        return {'choices': [{'text': response}]}
+
+    def get_provider_default_settings(self):
+        return self.model.model_kwargs
+
 def get_model(temperature, top_p, repetition_penalty):
     return HuggingFaceHub(
         repo_id="mistralai/Mistral-7B-Instruct-v0.3",
@@ -102,14 +109,13 @@ def respond(
     if model is None:
         logging.error("Model is None. Please select a valid model.")
         return "Error: No model selected. Please choose a valid model."
-
-    global llm
-    global llm_model
+    
     chat_template = get_messages_formatter_type(model)
-    if llm is None or llm_model != model:
-        llm = get_model(temperature, top_p, repeat_penalty)
-        llm_model = model
-    provider = LlamaCppPythonProvider(llm)
+    
+    # Create a new model instance for each request
+    llm = get_model(temperature, top_p, repeat_penalty)
+    
+    provider = HuggingFaceHubProvider(llm)
     logging.info(f"Loaded chat examples: {chat_template}")
     search_tool = WebSearchTool(
         llm_provider=provider,
@@ -133,12 +139,12 @@ def respond(
     )
 
     settings = provider.get_provider_default_settings()
-    settings.stream = False
-    settings.temperature = temperature
-    settings.top_k = top_k
-    settings.top_p = top_p
-    settings.max_tokens = max_tokens
-    settings.repeat_penalty = repeat_penalty
+    settings['stream'] = False
+    settings['temperature'] = temperature
+    settings['top_k'] = top_k
+    settings['top_p'] = top_p
+    settings['max_tokens'] = max_tokens
+    settings['repeat_penalty'] = repeat_penalty
 
     output_settings = LlmStructuredOutputSettings.from_functions(
         [search_tool.get_tool()]
@@ -163,7 +169,7 @@ def respond(
 
     outputs = ""
 
-    settings.stream = True
+    settings['stream'] = True
     response_text = answer_agent.get_chat_response(
         f"Write a detailed and complete research document that fulfills the following user request: '{message}', based on the information from the web below.\n\n" +
         result[0]["return_value"],