Update app.py

app.py

@@ -5,9 +5,11 @@ import base64
 import datetime
 import traceback
 import asyncio
+import multiprocessing
 from flask import Flask, request, jsonify
 from num2words import num2words
-from transformers import pipeline
+from huggingface_hub import hf_hub_download
+from llama_cpp import Llama
 
 # ══════════════════════════════════════════
 # CONFIG
@@ -42,7 +44,7 @@ def clean_text_for_tts(text):
     return text
 
 # ══════════════════════════════════════════
-# LOAD UNSLOTH GGUF & EDGE-TTS
+# LOAD UNSLOTH GGUF (via llama.cpp) & EDGE-TTS
 # ══════════════════════════════════════════
 print("=" * 55)
 print("  J.A.R.V.I.S. — Booting Systems")
@@ -51,20 +53,23 @@ print("=" * 55)
 LLM_ID = "unsloth/LFM2.5-1.2B-Instruct-GGUF"
 GGUF_FILE = "LFM2.5-1.2B-Instruct-UD-Q8_K_XL.gguf"
 
-print(f"[1/2] Loading {GGUF_FILE} via pipeline...")
+print(f"[1/2] Downloading & Loading {GGUF_FILE} via llama.cpp...")
 try:
-    # Explicitly load the exact GGUF file requested
-    pipe = pipeline(
-        "text-generation", 
-        model=LLM_ID, 
-        device_map="cpu", 
-        model_kwargs={"gguf_file": GGUF_FILE}
+    # Auto-downloads the specific GGUF file from the repo and caches it
+    model_path = hf_hub_download(repo_id=LLM_ID, filename=GGUF_FILE)
+    
+    # Initialize Llama.cpp engine
+    llm = Llama(
+        model_path=model_path,
+        n_ctx=4096,                           # Context window
+        n_threads=multiprocessing.cpu_count(),# Maximize CPU usage
+        verbose=False                         # Disable spammy C++ logs
     )
-    print(f"  ✅ {LLM_ID} loaded with {GGUF_FILE}!")
+    print(f"  ✅ {GGUF_FILE} loaded successfully!")
 except Exception as e:
     print(f"  ❌ Model FAILED completely: {e}")
     traceback.print_exc()
-    raise SystemExit("Cannot start without LLM. Check HF_TOKEN and GGUF compatibility.")
+    raise SystemExit("Cannot start without LLM.")
 
 print("[2/2] Loading edge-tts...")
 try:
@@ -75,7 +80,7 @@ except ImportError as e:
     edge_tts = None
 
 print("=" * 55)
-print(f"  LLM  : {GGUF_FILE}")
+print(f"  LLM  : {GGUF_FILE} (llama.cpp)")
 print(f"  TTS  : edge-tts ({'READY' if edge_tts else 'DISABLED'})")
 print(f"  Voice: {TTS_VOICE} | Rate: +7% | Pitch: +20Hz")
 print(f"  Max tokens: {MAX_NEW_TOKENS}")
@@ -107,7 +112,7 @@ def add_to_memory(sid, role, content):
 def generate_response(user_input, session_id):
     memory = get_memory(session_id)
 
-    # Build chat messages
+    # Build chat messages for llama.cpp chat templates
     messages =[
         {"role": "system", "content": SYSTEM_PROMPT},
         {"role": "assistant", "content": "I am waiting for you!"},
@@ -122,18 +127,17 @@ def generate_response(user_input, session_id):
     # Current user message
     messages.append({"role": "user", "content": user_input})
 
-    # Generate via pipeline
-    outputs = pipe(
-        messages,
-        max_new_tokens=MAX_NEW_TOKENS,
-        do_sample=True,
+    # Generate via llama-cpp-python
+    output = llm.create_chat_completion(
+        messages=messages,
+        max_tokens=MAX_NEW_TOKENS,
         temperature=0.9,
         top_k=45,
         top_p=0.97,
     )
 
     # Extract the assistant's newly generated text
-    response = outputs[0]["generated_text"][-1]["content"].strip()
+    response = output['choices'][0]['message']['content'].strip()
 
     if not response or len(response) < 2:
         response = "I appear to have momentarily lost my train of thought. Could you rephrase that?"
@@ -147,7 +151,7 @@ def generate_response(user_input, session_id):
 # ══════════════════════════════════════════
 async def _synthesize_edge(text, voice):
     # Applied specific settings from your image UI: rate +7%, pitch +20Hz
-    communicate = edge_tts.Communicate(text, voice, rate="+9%", pitch="+15Hz")
+    communicate = edge_tts.Communicate(text, voice, rate="+7%", pitch="+20Hz")
     audio_data = b""
     async for chunk in communicate.stream():
         if chunk["type"] == "audio":
@@ -584,7 +588,7 @@ def clear():
 def health():
     return jsonify({
         "status": "online",
-        "llm": "unsloth/LFM2.5-1.2B-Instruct-GGUF",
+        "llm": "unsloth/LFM2.5-1.2B-Instruct-GGUF (llama.cpp)",
         "tts_mode": "edge-tts",
         "tts_model": "edge-tts" if edge_tts else "DISABLED",
         "tts_voice": TTS_VOICE,