Upload 2 files

app.py

@@ -0,0 +1,266 @@
+import os
+import time
+from functools import lru_cache
+
+import gradio as gr
+from huggingface_hub import hf_hub_download
+from llama_cpp import Llama
+from transformers import AutoTokenizer
+
+MODEL_REPO = os.getenv("MODEL_REPO", "HuggingFaceTB/SmolLM2-1.7B-Instruct-GGUF")
+MODEL_FILE = os.getenv("MODEL_FILE", "smollm2-1.7b-instruct-q4_k_m.gguf")
+TOKENIZER_REPO = os.getenv("TOKENIZER_REPO", "HuggingFaceTB/SmolLM2-1.7B-Instruct")
+MAX_TOTAL_TOKENS = int(os.getenv("MAX_TOTAL_TOKENS", "2048"))
+DEFAULT_MAX_NEW_TOKENS = int(os.getenv("DEFAULT_MAX_NEW_TOKENS", "220"))
+DEFAULT_TEMPERATURE = float(os.getenv("DEFAULT_TEMPERATURE", "0.55"))
+DEFAULT_TOP_P = float(os.getenv("DEFAULT_TOP_P", "0.9"))
+DEFAULT_TOP_K = int(os.getenv("DEFAULT_TOP_K", "40"))
+DEFAULT_REPEAT_PENALTY = float(os.getenv("DEFAULT_REPEAT_PENALTY", "1.08"))
+DEFAULT_SYSTEM = os.getenv(
+    "DEFAULT_SYSTEM",
+    "You are concise, sharp, and helpful. Follow the system instruction carefully, but do not become robotic or overly cautious. Answer directly.",
+)
+
+CSS = """
+:root {
+  --bg-0: #07111f;
+  --bg-1: #0d1b2f;
+  --bg-2: #10233c;
+  --text: #ebf3ff;
+  --muted: #a9bdd9;
+  --accent: #7cc7ff;
+  --accent-2: #9b8cff;
+  --card: rgba(255,255,255,0.06);
+  --border: rgba(255,255,255,0.11);
+  --shadow: 0 18px 60px rgba(0,0,0,0.35);
+}
+body, .gradio-container {
+  background:
+    radial-gradient(70rem 40rem at 10% -10%, rgba(124,199,255,0.18), transparent 55%),
+    radial-gradient(50rem 35rem at 100% 0%, rgba(155,140,255,0.14), transparent 40%),
+    linear-gradient(180deg, var(--bg-0), var(--bg-1) 50%, #091320);
+  color: var(--text);
+}
+.top-wrap {
+  max-width: 1100px;
+  margin: 0 auto;
+}
+.hero {
+  border: 1px solid var(--border);
+  background: linear-gradient(180deg, rgba(255,255,255,0.08), rgba(255,255,255,0.045));
+  backdrop-filter: blur(12px);
+  border-radius: 26px;
+  padding: 28px 28px 22px 28px;
+  box-shadow: var(--shadow);
+  margin-bottom: 18px;
+}
+.hero h1 {
+  font-size: 34px;
+  line-height: 1.05;
+  margin: 0;
+  letter-spacing: -0.03em;
+}
+.hero p {
+  color: var(--muted);
+  margin: 12px 0 0 0;
+  font-size: 15px;
+}
+.card {
+  border: 1px solid var(--border) !important;
+  background: linear-gradient(180deg, rgba(255,255,255,0.07), rgba(255,255,255,0.045)) !important;
+  backdrop-filter: blur(12px);
+  border-radius: 24px !important;
+  box-shadow: var(--shadow);
+}
+.pill {
+  display: inline-flex;
+  gap: 8px;
+  align-items: center;
+  padding: 9px 12px;
+  border-radius: 999px;
+  border: 1px solid var(--border);
+  background: rgba(255,255,255,0.04);
+  color: var(--muted);
+  font-size: 12px;
+  margin-right: 8px;
+}
+#run-btn {
+  background: linear-gradient(135deg, var(--accent), var(--accent-2)) !important;
+  color: #08111d !important;
+  border: 0 !important;
+  font-weight: 700 !important;
+  min-height: 52px !important;
+  border-radius: 18px !important;
+}
+#stop-btn {
+  min-height: 52px !important;
+  border-radius: 18px !important;
+}
+.output-shell {
+  min-height: 420px;
+}
+.output-shell textarea, .output-shell .wrap {
+  font-size: 15px !important;
+  line-height: 1.6 !important;
+}
+.footer-note {
+  color: var(--muted);
+  font-size: 12px;
+  text-align: center;
+  margin-top: 8px;
+}
+"""
+
+@lru_cache(maxsize=1)
+def get_tokenizer():
+    return AutoTokenizer.from_pretrained(TOKENIZER_REPO)
+
+@lru_cache(maxsize=1)
+def get_model():
+    model_path = hf_hub_download(repo_id=MODEL_REPO, filename=MODEL_FILE)
+    cpu_count = os.cpu_count() or 2
+    threads = max(1, min(cpu_count, 8))
+    return Llama(
+        model_path=model_path,
+        n_ctx=MAX_TOTAL_TOKENS,
+        n_threads=threads,
+        n_threads_batch=threads,
+        n_batch=256,
+        n_ubatch=256,
+        use_mmap=True,
+        use_mlock=False,
+        flash_attn=False,
+        logits_all=False,
+        verbose=False,
+        seed=42,
+    )
+
+
+def build_prompt(system_prompt: str, user_prompt: str) -> str:
+    tokenizer = get_tokenizer()
+    messages = [
+        {"role": "system", "content": system_prompt.strip()},
+        {"role": "user", "content": user_prompt.strip()},
+    ]
+    return tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+
+
+def format_metrics(start_time: float, first_token_time: float | None, output_text: str) -> str:
+    total = max(time.time() - start_time, 1e-6)
+    first = None if first_token_time is None else first_token_time - start_time
+    chars = len(output_text)
+    cps = chars / total
+    rows = [
+        ["Total time", f"{total:.2f}s"],
+        ["Time to first token", "-" if first is None else f"{first:.2f}s"],
+        ["Characters", str(chars)],
+        ["Chars/sec", f"{cps:.1f}"],
+    ]
+    table = "<table style='width:100%; border-collapse:collapse;'>"
+    for k, v in rows:
+        table += f"<tr><td style='padding:8px 10px; color:#a9bdd9; border-bottom:1px solid rgba(255,255,255,0.08);'>{k}</td><td style='padding:8px 10px; text-align:right; border-bottom:1px solid rgba(255,255,255,0.08);'>{v}</td></tr>"
+    table += "</table>"
+    return table
+
+
+def generate(system_prompt, user_prompt, max_new_tokens, temperature, top_p, top_k, repeat_penalty):
+    if not user_prompt or not user_prompt.strip():
+        raise gr.Error("Kirjoita ensin prompti.")
+
+    model = get_model()
+    prompt = build_prompt(system_prompt, user_prompt)
+    max_new_tokens = int(max(32, min(max_new_tokens, 512)))
+    start_time = time.time()
+    first_token_time = None
+    text = ""
+
+    stream = model(
+        prompt,
+        max_tokens=max_new_tokens,
+        temperature=temperature,
+        top_p=top_p,
+        top_k=top_k,
+        repeat_penalty=repeat_penalty,
+        stop=["<|im_end|>", "<|endoftext|>"],
+        stream=True,
+    )
+
+    yield "", "", gr.update(interactive=False), gr.update(interactive=True)
+
+    for chunk in stream:
+        token = chunk["choices"][0]["text"]
+        if token:
+            if first_token_time is None:
+                first_token_time = time.time()
+            text += token
+            metrics = format_metrics(start_time, first_token_time, text)
+            yield text.strip(), metrics, gr.update(interactive=False), gr.update(interactive=True)
+
+    metrics = format_metrics(start_time, first_token_time, text)
+    yield text.strip(), metrics, gr.update(interactive=True), gr.update(interactive=False)
+
+
+def clear_all():
+    return "", "", ""
+
+
+with gr.Blocks(css=CSS, theme=gr.themes.Base(), fill_width=True) as demo:
+    with gr.Column(elem_classes=["top-wrap"]):
+        gr.HTML(
+            """
+            <div class='hero'>
+              <div class='pill'>SmolLM2 1.7B Instruct</div>
+              <div class='pill'>Q4_K_M GGUF</div>
+              <div class='pill'>CPU Basic friendly</div>
+              <h1>Fast local-feeling text generation.</h1>
+              <p>Minimal UI, fast start, streamed output, and a setup tuned for Hugging Face Spaces CPU Basic.</p>
+            </div>
+            """
+        )
+
+        with gr.Row(equal_height=True):
+            with gr.Column(scale=11):
+                with gr.Group(elem_classes=["card"]):
+                    system_box = gr.Textbox(
+                        label="System instruction",
+                        value=DEFAULT_SYSTEM,
+                        lines=4,
+                        max_lines=8,
+                        container=True,
+                    )
+                    prompt_box = gr.Textbox(
+                        label="Prompt",
+                        placeholder="Write your request here...",
+                        lines=10,
+                        max_lines=16,
+                        container=True,
+                    )
+                    with gr.Row():
+                        run_btn = gr.Button("Generate", elem_id="run-btn")
+                        stop_btn = gr.Button("Stop", elem_id="stop-btn")
+                        clear_btn = gr.Button("Clear")
+                    with gr.Accordion("Tuning", open=False):
+                        max_new_tokens = gr.Slider(64, 512, value=DEFAULT_MAX_NEW_TOKENS, step=8, label="Max new tokens")
+                        temperature = gr.Slider(0.0, 1.4, value=DEFAULT_TEMPERATURE, step=0.05, label="Temperature")
+                        top_p = gr.Slider(0.1, 1.0, value=DEFAULT_TOP_P, step=0.05, label="Top-p")
+                        top_k = gr.Slider(1, 100, value=DEFAULT_TOP_K, step=1, label="Top-k")
+                        repeat_penalty = gr.Slider(1.0, 1.3, value=DEFAULT_REPEAT_PENALTY, step=0.01, label="Repeat penalty")
+
+            with gr.Column(scale=10):
+                with gr.Group(elem_classes=["card", "output-shell"]):
+                    output_box = gr.Textbox(label="Output", lines=22, max_lines=22, show_copy_button=True)
+                    metrics_box = gr.HTML()
+
+        gr.HTML("<div class='footer-note'>For best CPU latency, keep max new tokens moderate and system prompts short.</div>")
+
+    generation = run_btn.click(
+        fn=generate,
+        inputs=[system_box, prompt_box, max_new_tokens, temperature, top_p, top_k, repeat_penalty],
+        outputs=[output_box, metrics_box, run_btn, stop_btn],
+        show_progress="hidden",
+    )
+    stop_btn.click(fn=None, cancels=[generation])
+    clear_btn.click(fn=clear_all, outputs=[prompt_box, output_box, metrics_box], show_progress="hidden")
+
+if __name__ == "__main__":
+    demo.queue(max_size=8, default_concurrency_limit=1).launch()

requirements.txt

@@ -0,0 +1,5 @@
+gradio>=5.23.3
+huggingface_hub>=0.30.0
+llama-cpp-python>=0.3.7
+transformers>=4.51.0
+sentencepiece>=0.2.0