Update app.py

Second change ,
changed the llm model to - Kutches/UncensoredV2 (2B, GGUF, multiple quant levels) . Unrestricted , with Q4_K_M ≈ 1.4 - 2.0 GB that fits comfortably and performs well on CPU Basic.

app.py

@@ -1,69 +1,71 @@
+import os, threading
 import gradio as gr
 from llama_cpp import Llama
-import os
 
-# GGUF model on the Hub (4-bit). You can change to another GGUF filename from the same repo if needed.
-REPO_ID = "Melvin56/Nidum-Llama-3.2-3B-Uncensored-IQ4_XS-GGUF"
-# Use a wildcard or exact filename; IQ4_XS is very light for CPU Basic.
-FILENAME_PATTERN = "*Q4_K_M.gguf"  # fallback wildcard
-N_CTX = int(os.getenv("N_CTX", "4096"))  # context length; keep moderate on free CPU
+# ====== MODEL CHOICE (uncensored, 2B, GGUF) ======
+REPO_ID = "Kutches/UncensoredV2"
+# Use a 4-bit file for free CPU. Q4_K_M is a great balance of speed/quality.
+# llama-cpp-python supports glob patterns for filename.
+FILENAME_PATTERN = "*Q4_K_M*.gguf"  # will match the Q4_K_M file in the repo
 
-# Download and memory-map the model from the Hub
-# This API pulls GGUF directly; no GPU is required.
+# ====== RUNTIME SETTINGS (tune for stability on 2 vCPU/16 GB) ======
+N_CTX   = int(os.getenv("N_CTX", "2048"))    # reduce to 2048 for memory headroom
+N_THREADS = None                              # Let llama.cpp pick; or set to 2
+# If you see slow tokenization, you can set N_THREADS=2 explicitly.
+
+# ====== LOAD MODEL (downloads from the Hub automatically) ======
 llm = Llama.from_pretrained(
     repo_id=REPO_ID,
-    filename=FILENAME_PATTERN,
+    filename=FILENAME_PATTERN,   # glob is supported
     n_ctx=N_CTX,
-    n_threads=None,      # let llama-cpp choose based on CPU cores
+    n_threads=N_THREADS,
     verbose=False
 )
 
-SYSTEM_DEFAULT = "You are a helpful assistant. Keep replies concise."
+SYSTEM_DEFAULT = "You are a helpful assistant. Answer clearly and concisely."
 
-def format_messages(messages, system_prompt):
-    # llama.cpp expects a single prompt string. We'll compose a simple chat format.
-    # You can switch to Alpaca-style or other templates if a GGUF requires it.
+def build_prompt(messages, system_prompt=None):
+    # Simple instruction-style prompt works reliably with many GGUF finetunes
     sys = system_prompt or SYSTEM_DEFAULT
-    conv = f"<<SYS>>\n{sys}\n<</SYS>>\n"
+    prompt = f"<<SYS>>\n{sys}\n<</SYS>>\n"
     for m in messages:
         role = m.get("role", "user")
-        content = m.get("content", "")
+        content = (m.get("content") or "").strip()
         if role == "user":
-            conv += f"[INST] {content.strip()} [/INST]\n"
+            prompt += f"[INST] {content} [/INST]\n"
         elif role == "assistant":
-            conv += f"{content.strip()}\n"
-    return conv.strip()
+            prompt += content + "\n"
+    return prompt.strip()
 
-def generate_reply(messages, temperature, top_p, max_tokens):
-    prompt = format_messages(messages, system_prompt=None)
-    # Streaming responses with llama.cpp
+def stream_reply(messages, temperature=0.7, top_p=0.9, max_tokens=384):
+    prompt = build_prompt(messages)
     stream = llm(
         prompt,
         max_tokens=max_tokens,
         temperature=temperature,
         top_p=top_p,
         stop=["</s>", "[/INST]"],
-        stream=True
+        stream=True,
     )
-    partial = ""
+    buf = ""
     for chunk in stream:
-        token = chunk.get("choices", [{}])[0].get("text", "")
-        partial += token
-        yield partial
+        text = chunk.get("choices", [{}])[0].get("text", "")
+        buf += text
+        yield buf
 
-with gr.Blocks(title="Nidum-Llama-3.2-3B-Uncensored (CPU)") as demo:
-    gr.Markdown("## Nidum-Llama-3.2-3B-Uncensored (GGUF on CPU Basic)")
+with gr.Blocks(title="Uncensored 2B (CPU Free Tier)") as demo:
+    gr.Markdown("### Uncensored 2B on Hugging Face Free Tier (CPU)")
     chat = gr.Chatbot(type="messages")
     with gr.Row():
         temp = gr.Slider(0.0, 1.5, 0.7, label="temperature")
         topp = gr.Slider(0.0, 1.0, 0.9, label="top_p")
         max_new = gr.Slider(32, 1024, 384, step=16, label="max_new_tokens")
-    inp = gr.Textbox(placeholder="Ask something...", label="Message")
+    msg = gr.Textbox(placeholder="Ask anything…", label="Message")
 
-    def respond(msg, history, temperature, top_p, max_new_tokens):
-        history = (history or []) + [{"role": "user", "content": msg}]
-        return generate_reply(history, temperature, top_p, max_new_tokens)
+    def respond(m, history, temperature, top_p, max_new_tokens):
+        history = (history or []) + [{"role": "user", "content": m}]
+        return stream_reply(history, temperature, top_p, max_new_tokens)
 
-    inp.submit(respond, [inp, chat, temp, topp, max_new], chat)
+    msg.submit(respond, [msg, chat, temp, topp, max_new], chat)
 
-demo.queue().launch()
+demo.queue().launch()