Update app.py

app.py

@@ -1,35 +1,67 @@
+import os
 import gradio as gr
 from llama_cpp import Llama
 from huggingface_hub import hf_hub_download
 
-print("Loading 1.5B Model...")
+# ──────────────────────────────────────────────
+#  MODEL LOADING  (cached to /tmp to skip re-download on warm restarts)
+# ──────────────────────────────────────────────
 
-model_path = hf_hub_download(
-    repo_id="Qwen/Qwen2.5-Coder-1.5B-Instruct-GGUF",
-    filename="qwen2.5-coder-1.5b-instruct-q8_0.gguf"
-)
+MODEL_CACHE = "/tmp/qwen_coder.gguf"
+
+print("Checking model cache...")
+if not os.path.exists(MODEL_CACHE):
+    print("Downloading Q4_K_M model (first run only)...")
+    model_path = hf_hub_download(
+        repo_id="Qwen/Qwen2.5-Coder-1.5B-Instruct-GGUF",
+        filename="qwen2.5-coder-1.5b-instruct-q4_k_m.gguf",  # ~2x faster than Q8
+    )
+    os.rename(model_path, MODEL_CACHE)
+    print("Model cached.")
+else:
+    print("Model found in cache, skipping download.")
 
+print("Loading model into memory...")
 llm = Llama(
-    model_path=model_path,
-    n_ctx=1024,
-    n_threads=2,
-    n_batch=512
+    model_path=MODEL_CACHE,
+    n_ctx=512,          # Reduced from 1024 — context scaling costs CPU cycles
+    n_threads=2,        # Matches HuggingFace free tier vCPU count
+    n_batch=128,        # Lower batch = less memory pressure on CPU
+    n_gpu_layers=0,     # No GPU on free tier
+    use_mlock=False,    # Don't lock memory on free tier
+    verbose=False,      # Skip logging overhead
 )
+print("Model ready!")
+
+# ──────────────────────────────────────────────
+#  INFERENCE
+# ──────────────────────────────────────────────
+
+SYSTEM_PROMPT = "You are a helpful Minecraft coding assistant. You help with Minecraft plugin development, mods, datapacks, and commands. Be concise and practical."
 
 def respond(message, history):
-    prompt = f"<|im_start|>system\nYou are a Minecraft coding assistant.<|im_end|>\n"
-    for user_msg, assistant_msg in history:
-        prompt += f"<|im_start|>user\n{user_msg}<|im_end|>\n<|im_start|>assistant\n{assistant_msg}<|im_end|>\n"
+    # Cap history to last 3 exchanges to prevent prompt bloat
+    history = history[-3:] if history else []
+
+    # Build prompt in ChatML format
+    prompt = f"<|im_start|>system\n{SYSTEM_PROMPT}<|im_end|>\n"
+    for exchange in history:
+        user_msg = exchange[0] if isinstance(exchange, (list, tuple)) else exchange.get("role", "")
+        asst_msg = exchange[1] if isinstance(exchange, (list, tuple)) else ""
+        if user_msg:
+            prompt += f"<|im_start|>user\n{user_msg}<|im_end|>\n"
+        if asst_msg:
+            prompt += f"<|im_start|>assistant\n{asst_msg}<|im_end|>\n"
     prompt += f"<|im_start|>user\n{message}<|im_end|>\n<|im_start|>assistant\n"
 
     response_text = ""
-    # Setting min_p and temperature for faster but logical output
     stream = llm(
         prompt,
-        max_tokens=256,
-        stop=["<|im_end|>", "user\n"],
+        max_tokens=128,         # Halves worst-case wait vs 256
+        temperature=0.7,
+        repeat_penalty=1.1,     # Prevents repetition loops that waste tokens
+        stop=["<|im_end|>", "<|im_start|>"],
         stream=True,
-        temperature=0.7
     )
 
     for chunk in stream:
@@ -37,6 +69,21 @@ def respond(message, history):
         response_text += token
         yield response_text
 
-demo = gr.ChatInterface(respond)
+# ──────────────────────────────────────────────
+#  UI
+# ──────────────────────────────────────────────
+
+demo = gr.ChatInterface(
+    fn=respond,
+    title="⛏️ Minecraft Coding Assistant",
+    description="Powered by Qwen2.5-Coder 1.5B · Q4_K_M · CPU optimized",
+    examples=[
+        "How do I create a custom recipe in a datapack?",
+        "Write a Spigot plugin that teleports a player on command",
+        "What's the syntax for a /execute command with conditions?",
+    ],
+    cache_examples=False,
+)
+
 if __name__ == "__main__":
     demo.launch()