Rename Jeppapp.py to app.py

Jeppapp.py → app.py

@@ -1,29 +1,36 @@
 import gradio as gr
-import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer
+import subprocess
+from huggingface_hub import hf_hub_download
 
-MODEL_ID = "Jeppcode/ScalableLab2"
-SUBFOLDER = "merged-model-fp16"
+# 1. Install llama-cpp-python i runtime (inte via requirements.txt)
+#    Viktigt: ta bort `llama-cpp-python` från requirements.txt,
+#    annars försöker Spaces bygga från källkod och fastnar.
+subprocess.run("pip install -q 'llama_cpp_python==0.3.15'", shell=True, check=False)
+from llama_cpp import Llama
 
-print(f"Loading model {MODEL_ID}/{SUBFOLDER} ...")
 
-# Tokenizer
-tokenizer = AutoTokenizer.from_pretrained(
-    MODEL_ID,
-    subfolder=SUBFOLDER,
+# 2. Ladda din GGUF-modell från Hugging Face
+MODEL_REPO = "Jeppcode/ScalableLab2"
+GGUF_FILENAME = "model-q4_k_m.gguf"  # eller "model-f16.gguf" om du vill ha fp16-varianten
+
+print(f"Downloading GGUF model {MODEL_REPO}/{GGUF_FILENAME} ...")
+model_path = hf_hub_download(
+    repo_id=MODEL_REPO,
+    filename=GGUF_FILENAME,
 )
 
-# Modell – fp16 och snålare på CPU
-model = AutoModelForCausalLM.from_pretrained(
-    MODEL_ID,
-    subfolder=SUBFOLDER,
-    dtype=torch.float16,        # samma som torch_dtype men utan varningen
-    low_cpu_mem_usage=True,
-    device_map="cpu",           # var explicit, allt på CPU
+print("Initializing llama.cpp LLM ...")
+llm = Llama(
+    model_path=model_path,
+    n_ctx=2048,      # kontextlängd
+    n_threads=2,     # trådar (Spaces CPU är begränsad)
+    n_batch=64,      # batchstorlek för generation
+    use_mmap=True,
+    use_mlock=False,
 )
-model.eval()
 
-# Några stil-lägen som systemprompter
+
+# 3. Några stil-lägen som "system prompts"
 STYLE_SYSTEM_PROMPTS = {
     "Default": "You are a helpful, polite assistant.",
     "Short answer": (
@@ -40,99 +47,102 @@ STYLE_SYSTEM_PROMPTS = {
 }
 
 
-def build_prompt(message, history, style):
+def _extract_text_from_content(content):
     """
-    I Gradio 6 är history en lista av dicts:
-    [
-      {"role": "user", "content": [...]},
-      {"role": "assistant", "content": [...]},
-      ...
-    ]
-    Vi plockar ut texten och mappar till {role, content}, och lägger till en systemprompt
-    baserat på vald 'style'.
+    Gradio 6 ChatInterface använder 'messages'-format.
+    content kan vara:
+      - en sträng
+      - en lista av blocks: [{"type": "text", "text": "..."} , ...]
+    Vi konverterar det till en enkel sträng.
     """
-    messages = []
+    if isinstance(content, list):
+        texts = []
+        for block in content:
+            if isinstance(block, dict) and block.get("type") == "text":
+                texts.append(block.get("text", ""))
+            else:
+                texts.append(str(block))
+        return "\n".join(t for t in texts if t)
+    else:
+        return str(content)
+
 
-    # Lägg till system / style prompt
+def build_prompt(message, history, style):
+    """
+    Bygger en enkel textprompt för llama.cpp baserat på:
+      - vald stil (system prompt)
+      - konversationshistorik
+      - senaste user-meddelandet
+    Vi använder ett simpelt format:
+        System: ...
+        Conversation:
+        User: ...
+        Assistant: ...
+        ...
+        User: <current message>
+        Assistant:
+    """
     system_prompt = STYLE_SYSTEM_PROMPTS.get(style, STYLE_SYSTEM_PROMPTS["Default"])
-    messages.append({"role": "system", "content": system_prompt})
 
-    # Tidigare historik
-    for msg in history:
+    prompt_parts = []
+    prompt_parts.append(f"System: {system_prompt}\n")
+    prompt_parts.append("Conversation:\n")
+
+    # history är en lista av dicts: {"role": "...", "content": ...}
+    for msg in history or []:
         role = msg.get("role")
-        content = msg.get("content", "")
-
-        # content kan vara en lista av blocks eller en sträng
-        if isinstance(content, list):
-            texts = []
-            for block in content:
-                if isinstance(block, dict) and block.get("type") == "text":
-                    texts.append(block.get("text", ""))
-                else:
-                    texts.append(str(block))
-            text = "\n".join(t for t in texts if t)
-        else:
-            text = str(content)
-
-        if text and role in ("user", "assistant", "system"):
-            messages.append({"role": role, "content": text})
-
-    # nuvarande användarmeddelande
-    messages.append({"role": "user", "content": message})
-
-    prompt = tokenizer.apply_chat_template(
-        messages,
-        tokenize=False,
-        add_generation_prompt=True,
-    )
-    return prompt
+        content = _extract_text_from_content(msg.get("content", ""))
 
+        if not content:
+            continue
 
-def chat_fn(message, history, max_new_tokens, temperature, top_p, repetition_penalty, style):
-    # Bygg prompt med historik + stil
-    prompt = build_prompt(message, history, style)
+        if role == "user":
+            prompt_parts.append(f"User: {content}\n")
+        elif role == "assistant":
+            prompt_parts.append(f"Assistant: {content}\n")
+        elif role == "system":
+            prompt_parts.append(f"System (previous): {content}\n")
+
+    # Nuvarande användarmeddelande
+    prompt_parts.append(f"User: {message}\n")
+    prompt_parts.append("Assistant:")
+
+    full_prompt = "".join(prompt_parts)
+    return full_prompt
 
-    inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
-
-    gen_kwargs = {
-        **inputs,
-        "max_new_tokens": int(max_new_tokens),
-        "pad_token_id": tokenizer.eos_token_id,
-        "eos_token_id": tokenizer.eos_token_id,
-        "repetition_penalty": float(repetition_penalty),
-    }
-
-    # Deterministisk om temperature == 0, annars sampling
-    if temperature <= 0.0:
-        gen_kwargs.update(
-            dict(
-                do_sample=False,
-                temperature=None,
-                top_p=None,
-            )
-        )
-    else:
-        gen_kwargs.update(
-            dict(
-                do_sample=True,
-                temperature=float(temperature),
-                top_p=float(top_p),
-            )
-        )
 
-    with torch.no_grad():
-        outputs = model.generate(**gen_kwargs)
+def chat_fn(message, history, max_new_tokens, temperature, top_p, repetition_penalty, style):
+    """
+    Huvudfunktionen som Gradio ChatInterface anropar.
+    - message: senaste user input
+    - history: tidigare meddelanden (messages-format)
+    - övriga parametrar: sliders / radio-knappar
+    """
+    prompt = build_prompt(message, history, style)
 
-    generated = tokenizer.decode(
-        outputs[0][inputs["input_ids"].shape[1]:],
-        skip_special_tokens=True,
-    ).strip()
+    # Hantera deterministiskt läge om temperature == 0
+    temp = float(temperature)
+    top_p_val = float(top_p)
+    repeat_pen = float(repetition_penalty)
+
+    if temp <= 0.0:
+        temp = 0.0
+        top_p_val = 1.0  # spelar mindre roll när temp=0
+
+    output = llm(
+        prompt,
+        max_tokens=int(max_new_tokens),
+        temperature=temp,
+        top_p=top_p_val,
+        repeat_penalty=repeat_pen,
+        stop=["User:", "Assistant:", "System:", "Conversation:"],
+    )
 
-    # ChatInterface sköter history själv, vi returnerar bara svaret
-    return generated
+    reply = output["choices"][0]["text"].strip()
+    return reply
 
 
-# DJ-reglagen (extra inputs till ChatInterface)
+# 4. DJ-reglagen (extra inputs till ChatInterface)
 max_new_tokens_slider = gr.Slider(
     minimum=16,
     maximum=256,
@@ -140,6 +150,7 @@ max_new_tokens_slider = gr.Slider(
     step=8,
     label="Max new tokens (response length)",
 )
+
 temperature_slider = gr.Slider(
     minimum=0.0,
     maximum=1.5,
@@ -147,6 +158,7 @@ temperature_slider = gr.Slider(
     step=0.1,
     label="Temperature (0 = deterministic, higher = more random)",
 )
+
 top_p_slider = gr.Slider(
     minimum=0.1,
     maximum=1.0,
@@ -154,6 +166,7 @@ top_p_slider = gr.Slider(
     step=0.05,
     label="Top-p (nucleus sampling)",
 )
+
 repetition_penalty_slider = gr.Slider(
     minimum=0.8,
     maximum=1.3,
@@ -161,6 +174,7 @@ repetition_penalty_slider = gr.Slider(
     step=0.05,
     label="Repetition penalty",
 )
+
 style_radio = gr.Radio(
     choices=[
         "Default",
@@ -174,10 +188,10 @@ style_radio = gr.Radio(
 
 demo = gr.ChatInterface(
     fn=chat_fn,
-    title="Lab 2 – Fine-tuned merged model (fp16)",
+    title="Lab 2 – Fine-tuned GGUF model",
     description=(
-        "Chat with our fine-tuned Llama-based model, merged to fp16 and "
-        "loaded from Jeppcode/ScalableLab2/merged-model-fp16.\n\n"
+        "Chat with our fine-tuned Llama-based model, converted to GGUF and "
+        "loaded via llama.cpp from Jeppcode/ScalableLab2.\n\n"
         "Use the controls in the accordion below like a DJ board to tweak "
         "response length, randomness and style."
     ),