feat: Add 3-tier quantization with auto hardware detection

- Tier 1: Qwen2.5-7B with 4-bit NF4 quantization (needs GPU 5.5GB+)
- Tier 2: Qwen2.5-1.5B with 8-bit quantization (GPU or CPU)
- Tier 3: Qwen2.5-0.5B bfloat16 fallback (CPU, default)
- Auto-detect GPU/CPU and select optimal quantization
- Auto-fallback: 7B→1.5B if GPU VRAM insufficient
- Configurable via .env: QUANTIZATION=auto/4bit/8bit/none
- Graceful fallback if bitsandbytes not installed
- Update requirements.txt with bitsandbytes>=0.43.0
- Update Dockerfile with GPU base image option

.env.example

@@ -4,3 +4,13 @@
 #   Qwen/Qwen2.5-1.5B-Instruct   (Goldilocks - ~3GB, recommended for GPU)
 #   Qwen/Qwen2.5-7B-Instruct     (Smartest - needs 4-bit quantization + GPU)
 MODEL_ID=Qwen/Qwen2.5-0.5B-Instruct
+
+# Quantization mode: auto, 4bit, 8bit, none
+#   auto = 4-bit on GPU for 7B, 8-bit on GPU for 1.5B, bf16 on CPU
+#   4bit = force 4-bit quantization (requires GPU + bitsandbytes)
+#   8bit = force 8-bit quantization
+#   none = disable quantization, uses bfloat16
+QUANTIZATION=auto
+
+# Double quantization for 4-bit (saves ~10% more memory)
+USE_DOUBLE_QUANT=true

Dockerfile

@@ -1,5 +1,10 @@
+# CPU-optimized default image (works on HuggingFace free tier)
 FROM python:3.9
 
+# For GPU support (uncomment to use):
+# FROM pytorch/pytorch:2.1.0-cuda12.1-cudnn8-runtime
+# RUN pip install bitsandbytes>=0.43.0
+
 RUN useradd -m -u 1000 user
 USER user
 ENV PATH="/home/user/.local/bin:$PATH"

requirements.txt

@@ -7,3 +7,4 @@ torch
 transformers
 accelerate
 sentencepiece
+bitsandbytes>=0.43.0

services/model_loader.py

@@ -1,6 +1,6 @@
 import os
 import torch
-from transformers import pipeline
+from transformers import pipeline, BitsAndBytesConfig
 from dotenv import load_dotenv
 from pathlib import Path
 
@@ -8,25 +8,140 @@ env_path = Path(__file__).resolve().parent.parent / ".env"
 load_dotenv(dotenv_path=env_path)
 
 MODEL_ID = os.getenv("MODEL_ID", "Qwen/Qwen2.5-0.5B-Instruct")
+QUANTIZATION = os.getenv("QUANTIZATION", "auto")
+USE_DOUBLE_QUANT = os.getenv("USE_DOUBLE_QUANT", "true").lower() == "true"
 
 _pipe = None
+_current_model = None
 
-def get_pipe():
-    global _pipe
-    if _pipe is None:
-        print(f"⏳ Loading Local Model {MODEL_ID}...")
+def _log(msg: str):
+    print(f"[ModelLoader] {msg}")
+
+def _has_gpu() -> bool:
+    return torch.cuda.is_available()
+
+def _gpu_name() -> str:
+    if _has_gpu():
+        return torch.cuda.get_device_name(0)
+    return "None"
+
+def _gpu_memory_gb() -> float:
+    if _has_gpu():
         try:
+            return torch.cuda.get_device_properties(0).total_mem / 1e9
+        except:
+            return 0
+    return 0
+
+def _select_quantization() -> str:
+    """Auto-select quantization tier based on MODEL_ID and hardware."""
+    user_mode = QUANTIZATION.lower()
+
+    if user_mode == "none":
+        return "none"
+
+    if user_mode != "auto":
+        return user_mode
+
+    # Auto-detect: GPU with enough VRAM for requested model
+    if "7B" in MODEL_ID:
+        if _has_gpu() and _gpu_memory_gb() >= 5.5:
+            _log(f"7B model detected, GPU {_gpu_name()} ({_gpu_memory_gb():.1f}GB) — using 4-bit")
+            return "4bit"
+        _log("7B model requested but no GPU with 5.5GB+ VRAM — falling back to 1.5B 8-bit")
+        return "cpu_fallback_8bit"
+
+    if "1.5B" in MODEL_ID:
+        if _has_gpu():
+            _log(f"1.5B model detected, GPU available — using 8-bit")
+            return "8bit"
+        _log("1.5B model detected, CPU only — using bfloat16")
+        return "none"
+
+    return "none"
+
+def _build_model_kwargs(quant_mode: str) -> dict:
+    """Build pipeline kwargs based on quantization mode."""
+    kwargs = {
+        "trust_remote_code": True,
+    }
+
+    if quant_mode == "4bit":
+        kwargs["device_map"] = "auto"
+        kwargs["quantization_config"] = BitsAndBytesConfig(
+            load_in_4bit=True,
+            bnb_4bit_compute_dtype=torch.bfloat16,
+            bnb_4bit_use_double_quant=USE_DOUBLE_QUANT,
+            bnb_4bit_quant_type="nf4",
+        )
+        _log("⚡ 4-bit quantization enabled (NF4, double quant)")
+
+    elif quant_mode == "8bit":
+        kwargs["device_map"] = "auto"
+        kwargs["quantization_config"] = BitsAndBytesConfig(
+            load_in_8bit=True,
+        )
+        _log("⚡ 8-bit quantization enabled")
+
+    elif quant_mode == "cpu_fallback_8bit":
+        kwargs["device_map"] = "auto"
+        kwargs["quantization_config"] = BitsAndBytesConfig(
+            load_in_8bit=True,
+        )
+        _log("⚡ CPU fallback 8-bit for 1.5B model")
+
+    else:
+        kwargs["torch_dtype"] = torch.bfloat16
+        kwargs["device_map"] = "auto"
+        _log(f"📦 Loading {MODEL_ID} in bfloat16 (CPU-friendly)")
+
+    return kwargs
+
+def get_pipe():
+    global _pipe, _current_model
+
+    if _pipe is not None:
+        return _pipe
+
+    actual_model_id = MODEL_ID
+    quant_mode = _select_quantization()
+
+    # Handle CPU fallback for 7B → 1.5B
+    if quant_mode == "cpu_fallback_8bit":
+        actual_model_id = "Qwen/Qwen2.5-1.5B-Instruct"
+        _log(f"🔄 Fallback: loading {actual_model_id} instead of {MODEL_ID}")
+
+    _log(f"🚀 Loading {actual_model_id} (quantization: {quant_mode})")
+    _log(f"   Hardware: GPU={_gpu_name()}, VRAM={_gpu_memory_gb():.1f}GB, CUDA={_has_gpu()}")
+
+    try:
+        kwargs = _build_model_kwargs(quant_mode)
+        _pipe = pipeline(
+            "text-generation",
+            model=actual_model_id,
+            **kwargs
+        )
+        _current_model = actual_model_id
+        _log("✅ Model loaded successfully!")
+    except ImportError as e:
+        if "bitsandbytes" in str(e):
+            _log("❌ bitsandbytes not installed. Falling back to CPU bfloat16.")
             _pipe = pipeline(
                 "text-generation",
-                model=MODEL_ID,
+                model=actual_model_id,
                 torch_dtype=torch.bfloat16,
                 device_map="auto",
-                trust_remote_code=True
+                trust_remote_code=True,
             )
-            print("✅ Model Loaded Successfully!")
-        except Exception as e:
-            print(f"❌ Model Load Failed: {e}")
+            _current_model = actual_model_id
+            _log("✅ Model loaded with CPU fallback")
+        else:
+            _log(f"❌ Model load failed: {e}")
             _pipe = None
+    except Exception as e:
+        _log(f"❌ Model load failed: {e}")
+        _pipe = None
+
     return _pipe
 
 def generate_text(messages, temperature=0.3, max_new_tokens=2000):