Fix LLäMmlein 1B chat Space runtime

Dockerfile

@@ -1,18 +0,0 @@
-# Use an alias for the base image for easier updates
-FROM python:3.10 as base
-
-# Set the working directory
-WORKDIR /app
-
-# Install Python requirements
-COPY ./requirements.txt /app/
-RUN pip install --no-cache-dir --upgrade -r requirements.txt
-
-# Download model
-RUN wget -nv https://huggingface.co/LSX-UniWue/LLaMmlein_1B_alternative_formats/resolve/LLaMmlein_1B_chat_selected/LLaMmlein_1B_chat_selected.gguf -O model.gguf
-
-# Copy the rest of your application
-COPY . .
-
-# Command to run the application
-CMD ["python", "app.py"]

README.md

@@ -7,15 +7,16 @@ models:
 colorFrom: purple
 colorTo: yellow
 sdk: gradio
-sdk_version: 5.6.0
+sdk_version: 5.50.0
 python_version: '3.10'
 preload_from_hub:
 - >-
   LSX-UniWue/LLaMmlein_1B_alternative_formats LLaMmlein_1B_chat_selected.gguf
   7d97b69ae6910b5f317be2dbd5b4820d848c66b4
 pinned: true
+suggested_hardware: cpu-basic
 thumbnail: >-
   https://cdn-uploads.huggingface.co/production/uploads/6070431e1a4c4d313032558b/6_LoaV5O5bsLImOQ1oTh_.png
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -1,71 +1,92 @@
 import os
-import json
+from pathlib import Path
+
 import gradio as gr
+from huggingface_hub import hf_hub_download
 from llama_cpp import Llama
-import spaces
 
-# Get environment variables
-model_id = os.getenv('MODEL')
-quant = os.getenv('QUANT')
-chat_template = os.getenv('CHAT_TEMPLATE')
+MODEL_ID = os.getenv("MODEL", "LSX-UniWue/LLaMmlein_1B_chat_selected")
+MODEL_REPO_ID = os.getenv("MODEL_REPO_ID", "LSX-UniWue/LLaMmlein_1B_alternative_formats")
+MODEL_REVISION = os.getenv("MODEL_REVISION", "7d97b69ae6910b5f317be2dbd5b4820d848c66b4")
+MODEL_FILENAME = os.getenv("MODEL_FILENAME", "LLaMmlein_1B_chat_selected.gguf")
+QUANT = os.getenv("QUANT", "Q8_0/BF16 GGUF")
+N_CTX = int(os.getenv("N_CTX", "2048"))
+N_THREADS = int(os.getenv("N_THREADS", str(min(4, os.cpu_count() or 2))))
+MAX_TOKENS = int(os.getenv("MAX_TOKENS", "64"))
 
-# Interface variables
-model_name = model_id.split('/')[-1]
+model_name = MODEL_ID.split("/")[-1]
 title = f"🇩🇪 {model_name}"
-description = f"Chat with <a href=\"https://huggingface.co/{model_id}\">{model_name}</a> in GGUF format ({quant})!"
-
-print("find gguf file")
-import os
-from pathlib import Path
-
-# Get the Hugging Face cache directory
-hf_cache_dir = os.getenv("HF_HOME", str(Path.home() / ".cache" / "huggingface"))
+description = (
+    f"Chat with <a href=\"https://huggingface.co/{MODEL_ID}\">{model_name}</a> "
+    f"in GGUF format ({QUANT})."
+)
 
-# List all files in the Hugging Face cache directory
-for root, dirs, files in os.walk(hf_cache_dir):
-    for file in files:
-        print(os.path.join(root, file))
+print("resolving model file", flush=True)
+model_path = hf_hub_download(
+    repo_id=MODEL_REPO_ID,
+    filename=MODEL_FILENAME,
+    revision=MODEL_REVISION,
+)
+print(f"loading model from {Path(model_path).name}", flush=True)
 
+llm = Llama(
+    model_path=model_path,
+    n_ctx=N_CTX,
+    n_threads=N_THREADS,
+    n_batch=64,
+    verbose=False,
+)
 
-print("loading model")
-# Initialize the LLM
-llm = Llama(model_path="/home/user/.cache/huggingface/hub/models--LSX-UniWue--LLaMmlein_1B_alternative_formats/snapshots/7d97b69ae6910b5f317be2dbd5b4820d848c66b4/LLaMmlein_1B_chat_selected.gguf",
-            n_ctx=32768,
-            n_threads=2,
-            chat_format=chat_template)
+def iter_history_messages(history):
+    for entry in history or []:
+        if isinstance(entry, dict):
+            role = entry.get("role")
+            content = entry.get("content")
+            if role in {"user", "assistant"} and content:
+                yield {"role": role, "content": content}
+        else:
+            human, assistant = entry
+            if human:
+                yield {"role": "user", "content": human}
+            if assistant:
+                yield {"role": "assistant", "content": assistant}
 
-# Function for streaming chat completions
-@spaces.GPU
 def chat_stream_completion(message, history):
-    #messages_prompts = [{"role": "system", "content": system_prompt}]
-    messages_prompts = []
-    for human, assistant in history:
-        messages_prompts.append({"role": "user", "content": human})
-        messages_prompts.append({"role": "assistant", "content": assistant})
+    messages_prompts = list(iter_history_messages(history))
     messages_prompts.append({"role": "user", "content": message})
+    prompt_parts = [
+        "Du bist ein hilfreicher deutschsprachiger Assistent.",
+        "",
+    ]
+    for item in messages_prompts:
+        label = "Benutzer" if item["role"] == "user" else "Assistent"
+        prompt_parts.append(f"{label}: {item['content']}")
+    prompt_parts.append("Assistent:")
+    prompt = "\n".join(prompt_parts)
 
-    response = llm.create_chat_completion(
-        messages=messages_prompts,
+    response = llm.create_completion(
+        prompt=prompt,
+        max_tokens=MAX_TOKENS,
         repeat_penalty=1.1,
-        #temperature=0,
         stream=True,
-        stop=["<|im_end|>"]
+        stop=["\nBenutzer:", "\nAssistent:"],
     )
     message_repl = ""
     for chunk in response:
-        if len(chunk['choices'][0]["delta"]) != 0 and "content" in chunk['choices'][0]["delta"]:
-            message_repl = message_repl + chunk['choices'][0]["delta"]["content"]
+        text = chunk["choices"][0].get("text", "")
+        if text:
+            message_repl = message_repl + text
         yield message_repl
 
-print("starting gradio")
-# Gradio chat interface
+print("starting gradio", flush=True)
 gr.ChatInterface(
     fn=chat_stream_completion,
+    type="messages",
     title=title,
     description=description,
-    #additional_inputs=[gr.Textbox("Du bist ein hilfreicher Assistent.")],
-    #additional_inputs_accordion="📝 System prompt",
     examples=[
         ["Was weißt du über Würzburg?"],
-    ]
-).queue().launch()
+        ["Erkläre Quantencomputing in einfachen Worten."],
+    ],
+    cache_examples=False,
+).queue().launch()

requirements.txt

@@ -1,2 +1,3 @@
-llama-cpp-python
-#gradio
+--extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu
+llama-cpp-python==0.3.30
+huggingface_hub>=0.33,<1