Update app.py

app.py

@@ -1,55 +1,77 @@
-import gradio as gr
-from openai import OpenAI
 import os
+import shutil
+import gradio as gr
+from llama_cpp import Llama
+from huggingface_hub import hf_hub_download
 
-MODEL_ID = "llama3.1-8b"
+# ── Load model onto CPU ──
+MODEL_CACHE = "/tmp/qwen_coder.gguf"
 
-# Retrieve the Cerebras key from Space Secrets
-CEREBRAS_API_KEY = os.getenv("CEREBRAS_API_KEY")
+if not os.path.exists(MODEL_CACHE) or os.path.getsize(MODEL_CACHE) < 1_000_000:
+    print("Downloading model...")
+    downloaded = hf_hub_download(
+        repo_id="Qwen/Qwen2.5-Coder-0.5B-Instruct-GGUF",
+        filename="qwen2.5-coder-0.5b-instruct-q4_k_m.gguf",
+    )
+    shutil.copy2(downloaded, MODEL_CACHE)
+    print("Model cached.")
+else:
+    print(f"Cache hit — {os.path.getsize(MODEL_CACHE)/1e9:.2f} GB")
 
+print("Loading model...")
+llm = Llama(
+    model_path=MODEL_CACHE,
+    n_ctx=512,
+    n_threads=2,
+    n_batch=32,
+    n_gpu_layers=0,
+    use_mlock=False,
+    verbose=False,
+)
+print("Model ready!")
+
+# ── Inference ──
 def respond(message, history, system_message, max_tokens, temperature, top_p):
-    # Initialize the Cerebras client (OpenAI-compatible)
-    client = OpenAI(
-        base_url="https://api.cerebras.ai/v1", 
-        api_key=CEREBRAS_API_KEY
-    )
-    
-    messages = [{"role": "system", "content": system_message}]
-    for user_msg, assistant_msg in history:
-        if user_msg: messages.append({"role": "user", "content": user_msg})
-        if assistant_msg: messages.append({"role": "assistant", "content": assistant_msg})
-    messages.append({"role": "user", "content": message})
-
-    response = ""
-    try:
-        stream = client.chat.completions.create(
-            model=MODEL_ID,
-            messages=messages,
-            max_tokens=max_tokens,
-            temperature=temperature,
-            top_p=top_p,
-            stream=True
-        )
-        
-        for chunk in stream:
-            if chunk.choices[0].delta.content:
-                token = chunk.choices[0].delta.content
-                response += token
-                yield response
-                
-    except Exception as e:
-        yield f"API Error: {str(e)}"
+    # Build ChatML prompt
+    prompt = f"<|im_start|>system\n{system_message}<|im_end|>\n"
+    for exchange in history[-3:]:
+        user_msg = exchange[0] if isinstance(exchange, (list, tuple)) else ""
+        asst_msg = exchange[1] if isinstance(exchange, (list, tuple)) else ""
+        if user_msg:
+            prompt += f"<|im_start|>user\n{user_msg}<|im_end|>\n"
+        if asst_msg:
+            prompt += f"<|im_start|>assistant\n{asst_msg}<|im_end|>\n"
+    prompt += f"<|im_start|>user\n{message}<|im_end|>\n<|im_start|>assistant\n"
+
+    response_text = ""
+    for chunk in llm(
+        prompt,
+        max_tokens=max_tokens,
+        temperature=temperature,
+        top_p=top_p,
+        top_k=20,
+        repeat_penalty=1.1,
+        stop=["<|im_end|>", "<|im_start|>"],
+        stream=True,
+    ):
+        response_text += chunk["choices"][0]["text"]
+        yield response_text
 
+# ── UI ──
 with gr.Blocks() as demo:
     gr.Markdown("# 🛠️ Minecraft Modding Log Analyzer")
-    
+
     with gr.Sidebar():
-        gr.LoginButton() 
+        gr.LoginButton()
         gr.Markdown("---")
-        system_msg = gr.Textbox(value="You are an Elite Minecraft Modder who fixes Fabric and Forge crash logs.", label="System Prompt")
-        tokens = gr.Slider(128, 8192, value=2048, label="Max Tokens")
-        temp = gr.Slider(0.1, 1.0, value=0.3, label="Temp")
-        top_p = gr.Slider(0.1, 1.0, value=0.9, label="Top-P")
+        system_msg = gr.Textbox(
+            value="You are an Elite Minecraft Modder who fixes Fabric and Forge crash logs.",
+            label="System Prompt",
+            lines=4,
+        )
+        tokens = gr.Slider(128, 2048, value=512, label="Max Tokens")
+        temp   = gr.Slider(0.1, 1.0,  value=0.3, label="Temp")
+        top_p  = gr.Slider(0.1, 1.0,  value=0.9, label="Top-P")
 
     gr.ChatInterface(
         respond,