Trying fix for interface

app.py

@@ -1,6 +1,21 @@
 import gradio as gr
-from huggingface_hub import InferenceClient
+from huggingface_hub import hf_hub_download
+from llama_cpp import Llama
 
+# Download the GGUF model from the new Organization Hub
+model_path = hf_hub_download(
+    repo_id="Emil-Matteus/llama-32-1b",
+    filename="llama-3.2-1b-instruct.Q4_K_M.gguf"
+)
+
+# Initialize the local Llama model
+# n_gpu_layers=0 forces CPU usage. n_ctx sets context window.
+llm = Llama(
+    model_path=model_path,
+    n_gpu_layers=0,
+    n_ctx=4096,
+    verbose=False
+)
 
 def respond(
     message,
@@ -9,36 +24,27 @@ def respond(
     max_tokens,
     temperature,
     top_p,
-    hf_token: gr.OAuthToken,
 ):
-    """
-    For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
-    """
-    client = InferenceClient(token=hf_token.token, model="Emil-Matteus/llama-32-1b")
-
     messages = [{"role": "system", "content": system_message}]
-
     messages.extend(history)
-
     messages.append({"role": "user", "content": message})
 
     response = ""
-
-    for message in client.chat_completion(
-        messages,
+    
+    # Generate response using the local model
+    completion = llm.create_chat_completion(
+        messages=messages,
         max_tokens=max_tokens,
-        stream=True,
         temperature=temperature,
         top_p=top_p,
-    ):
-        choices = message.choices
-        token = ""
-        if len(choices) and choices[0].delta.content:
-            token = choices[0].delta.content
-
-        response += token
-        yield response
+        stream=True
+    )
 
+    for chunk in completion:
+        if "content" in chunk["choices"][0]["delta"]:
+            token = chunk["choices"][0]["delta"]["content"]
+            response += token
+            yield response
 
 """
 For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
@@ -60,11 +66,5 @@ chatbot = gr.ChatInterface(
     ],
 )
 
-with gr.Blocks() as demo:
-    with gr.Sidebar():
-        gr.LoginButton()
-    chatbot.render()
-
-
 if __name__ == "__main__":
-    demo.launch()
+    chatbot.launch()

requirements.txt

@@ -0,0 +1,3 @@
+llama-cpp-python
+huggingface-hub
+gradio