feat: self-hosted Qwen2.5-1.5B-Instruct via transformers — no external API, no compilation

.env.example

@@ -1,12 +1,11 @@
 # hf-backend HuggingFace Space environment variables
-# Set these in your HF Space settings -> Variables and Secrets
+# Set these in HF Space -> Settings -> Variables and Secrets
 
 SUPABASE_URL=https://YOUR_PROJECT_REF.supabase.co
 # New projects (Nov 2025+): use your Secret key  ->  sb_secret_...
 SUPABASE_KEY=sb_secret_your_secret_key_here
 
-# HF_TOKEN is automatically injected by HF Spaces -- no manual setup needed.
-# The app uses the free HF Serverless Inference API (InferenceClient).
-# Default LLM: mistralai/Mistral-7B-Instruct-v0.3
-# To use a different model, add this variable:
-# HF_LLM_MODEL=HuggingFaceH4/zephyr-7b-beta
+# Optional: override the default self-hosted LLM model
+# Default: Qwen/Qwen2.5-1.5B-Instruct (~3 GB, ~5-10 tok/s on 2 vCPUs)
+# Faster/smaller: Qwen/Qwen2.5-0.5B-Instruct (~1 GB, ~20 tok/s)
+# LLM_MODEL=Qwen/Qwen2.5-1.5B-Instruct

Dockerfile

@@ -2,17 +2,18 @@ FROM python:3.12-slim
 
 WORKDIR /app
 
-# git is needed for huggingface_hub operations.
-# No build-essential/cmake needed -- no local LLM compilation.
-# Inference runs on HF's free serverless API via InferenceClient.
+# git is needed for huggingface_hub model downloads.
+# No cmake/build-essential needed -- no C++ compilation.
+# LLM runs locally via transformers (Qwen2.5-1.5B-Instruct, ~3 GB bfloat16).
 RUN apt-get update && apt-get install -y git \
     && rm -rf /var/lib/apt/lists/*
 
 COPY requirements.txt .
 
 # -- Step 1: CPU-only PyTorch ------------------------------------------------
-# sentence-transformers depends on torch. Pre-installing the CPU wheel (~190 MB)
-# prevents pip from resolving the default CUDA bundle (~3.5 GB).
+# sentence-transformers + transformers both need torch.
+# Pre-installing the CPU wheel (~190 MB) prevents pip from resolving the
+# default CUDA bundle (~3.5 GB) which would blow the build disk quota.
 RUN pip install torch --index-url https://download.pytorch.org/whl/cpu \
     --no-cache-dir
 

app.py

@@ -7,7 +7,7 @@ from supabase import create_client
 import uuid
 import os
 
-from model.loader import get_client, get_model_name, is_llm_ready
+from model.loader import get_llm, get_model_name, is_llm_ready
 from retrieval.embedder import get_model, embed_chunks, embed_query
 from retrieval.vectorstore import store_chunks, similarity_search
 from ingestion.parser import parse_file
@@ -40,16 +40,17 @@ def _supa():
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):
-    print("🚀 Starting up...")
-    # Warm up the embedding model (~2 s)
-    get_model()
-    print("  ✔ Embedding model ready", flush=True)
-    # Initialise HF Inference client (free, uses auto-injected HF_TOKEN)
+    import asyncio
+    print("\U0001f680 Starting up...", flush=True)
+    get_model()  # BGE-small embedding model (~2s)
+    print("  \u2714 Embedding model ready", flush=True)
+    # Load the LLM in a thread so the event loop stays responsive
+    loop = asyncio.get_event_loop()
     try:
-        get_client()
-        print(f"  ✔ HF Inference LLM ready ({get_model_name()})", flush=True)
+        await loop.run_in_executor(None, get_llm)
+        print(f"  \u2714 LLM ready ({get_model_name()})", flush=True)
     except Exception as exc:
-        print(f"  ⚠  HF Inference init warning: {exc}", flush=True)
+        print(f"  \u26a0  LLM load failed: {exc}", flush=True)
     print("✅ Ready", flush=True)
     yield
 

generation/llm.py

@@ -1,4 +1,7 @@
-from model.loader import get_client
+import torch
+from model.loader import get_tokenizer, get_llm
+from transformers import TextIteratorStreamer
+from threading import Thread
 from typing import Generator
 
 SYSTEM_PROMPT = """You are a precise document study assistant by Md Tusar Akon.
@@ -12,7 +15,8 @@ def stream_answer(
     context_chunks: list[str],
     thinking_mode: bool = False,
 ) -> Generator[str, None, None]:
-    client = get_client()
+    tokenizer = get_tokenizer()
+    model = get_llm()
     context = "\n\n---\n\n".join(context_chunks)
 
     messages = [
@@ -20,15 +24,35 @@ def stream_answer(
         {"role": "user", "content": f"Context:\n{context}\n\nQuestion: {query}"},
     ]
 
-    stream = client.chat_completion(
-        messages=messages,
-        max_tokens=600,
-        temperature=0.2,
-        top_p=0.95,
-        stream=True,
+    input_ids = tokenizer.apply_chat_template(
+        messages,
+        add_generation_prompt=True,
+        return_tensors="pt",
     )
 
-    for chunk in stream:
-        delta = chunk.choices[0].delta.content or ""
-        if delta:
-            yield delta
+    streamer = TextIteratorStreamer(
+        tokenizer,
+        skip_prompt=True,
+        skip_special_tokens=True,
+        timeout=120.0,
+    )
+
+    thread = Thread(
+        target=model.generate,
+        kwargs=dict(
+            input_ids=input_ids,
+            streamer=streamer,
+            max_new_tokens=512,
+            temperature=0.2,
+            do_sample=True,
+            top_p=0.95,
+            pad_token_id=tokenizer.eos_token_id,
+        ),
+        daemon=True,
+    )
+    thread.start()
+
+    for token in streamer:
+        yield token
+
+    thread.join(timeout=120)

generation/quiz.py

@@ -1,6 +1,7 @@
-from model.loader import get_client
+import torch
 import json
 import re
+from model.loader import get_tokenizer, get_llm
 
 QUIZ_PROMPT = """Based on the context below, generate exactly 3 multiple-choice quiz questions.
 Each question must test understanding of the content, not trivia.
@@ -21,17 +22,28 @@ Respond ONLY with a JSON array, no markdown, no explanation:
 
 
 def generate_quiz(context_chunks: list[str]) -> list[dict]:
-    client = get_client()
+    tokenizer = get_tokenizer()
+    model = get_llm()
     context = "\n\n".join(context_chunks[:3])
 
-    result = client.chat_completion(
-        messages=[{"role": "user", "content": QUIZ_PROMPT.format(context=context)}],
-        max_tokens=800,
-        temperature=0.4,
-        stream=False,
+    messages = [{"role": "user", "content": QUIZ_PROMPT.format(context=context)}]
+    input_ids = tokenizer.apply_chat_template(
+        messages,
+        add_generation_prompt=True,
+        return_tensors="pt",
     )
 
-    raw = result.choices[0].message.content or ""
+    with torch.no_grad():
+        output_ids = model.generate(
+            input_ids,
+            max_new_tokens=800,
+            temperature=0.4,
+            do_sample=True,
+            pad_token_id=tokenizer.eos_token_id,
+        )
+
+    new_tokens = output_ids[0][input_ids.shape[-1]:]
+    raw = tokenizer.decode(new_tokens, skip_special_tokens=True)
     raw = re.sub(r"```json|```", "", raw).strip()
 
     try:

model/loader.py

@@ -1,35 +1,61 @@
 """
-HuggingFace Inference API client -- 100% free, zero compilation.
-
-Uses huggingface_hub.InferenceClient which calls HF's hosted inference servers.
-HF_TOKEN is automatically injected by HF Spaces (no manual setup needed).
-To enable: Space Settings -> "Grant repository access to this Space" is ON by default.
-
-Optional env var: HF_LLM_MODEL (default: mistralai/Mistral-7B-Instruct-v0.3)
+Self-hosted LLM using transformers � zero external API, no C++ compilation.
+Model: Qwen/Qwen2.5-1.5B-Instruct (1.5B params, ~3 GB bfloat16, fits 16 GB RAM)
+Downloads ~3 GB on first boot then caches to disk for subsequent starts.
+Speed on 2 vCPUs: ~5-10 tok/s  ?  20-60 s per RAG answer.
+Override: set LLM_MODEL env var (e.g. Qwen/Qwen2.5-0.5B-Instruct for faster inference).
 """
 
 import os
-from huggingface_hub import InferenceClient
+import time
+import torch
+from transformers import AutoTokenizer, AutoModelForCausalLM
+
+MODEL_ID = os.environ.get("LLM_MODEL", "Qwen/Qwen2.5-1.5B-Instruct")
+
+_tokenizer: AutoTokenizer | None = None
+_llm: AutoModelForCausalLM | None = None
+_llm_ready: bool = False
+
+
+def _load() -> None:
+    global _tokenizer, _llm, _llm_ready
+    if _llm is not None:
+        return
+
+    t0 = time.time()
+    print(f"\n{'-'*60}", flush=True)
+    print(f"  Loading {MODEL_ID}", flush=True)
+    print(f"  First boot downloads ~3 GB then caches to disk.", flush=True)
+    print(f"{'-'*60}\n", flush=True)
+
+    _tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
+    print("  Tokenizer loaded.", flush=True)
+
+    _llm = AutoModelForCausalLM.from_pretrained(
+        MODEL_ID,
+        torch_dtype=torch.bfloat16,   # half RAM vs float32, safe on modern CPUs
+    )
+    _llm.eval()
+    _llm_ready = True
+    print(f"\n{'-'*60}", flush=True)
+    print(f"  {MODEL_ID} ready in {time.time()-t0:.1f}s", flush=True)
+    print(f"{'-'*60}\n", flush=True)
 
-_client: InferenceClient | None = None
 
-DEFAULT_MODEL = "mistralai/Mistral-7B-Instruct-v0.3"
+def get_tokenizer() -> AutoTokenizer:
+    _load()
+    return _tokenizer  # type: ignore
 
 
-def get_client() -> InferenceClient:
-    global _client
-    if _client is None:
-        token = os.environ.get("HF_TOKEN", "")
-        model = get_model_name()
-        _client = InferenceClient(model=model, token=token or None)
-        print(f"  Inference client ready -- model: {model}", flush=True)
-    return _client
+def get_llm() -> AutoModelForCausalLM:
+    _load()
+    return _llm  # type: ignore
 
 
 def get_model_name() -> str:
-    return os.environ.get("HF_LLM_MODEL", DEFAULT_MODEL)
+    return MODEL_ID
 
 
 def is_llm_ready() -> bool:
-    """Always ready -- HF Inference API needs no local warmup."""
-    return True
+    return _llm_ready