Update app.py

app.py

@@ -1,25 +1,35 @@
+import os
 import gradio as gr
 import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
+from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer, BitsAndBytesConfig
 from peft import PeftModel
 from threading import Thread
 
-# 1. Map both coordinates: The base model engine and your custom adapter layer
+# 1. Map both coordinates
 BASE_MODEL = "Qwen/Qwen2.5-Coder-3B-Instruct"
 ADAPTER_MODEL = "Cydercoder/qwen2.5-coder-3b"
 
 print("Loading official base tokenizer...")
 tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL)
 
-print("Loading public base model on CPU...")
+print("Configuring aggressive 4-bit CPU/GPU quantization parameters...")
+# This config compresses the weights from 32-bit down to 4-bit integers to fit in RAM
+quantization_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_compute_dtype=torch.float32,
+    bnb_4bit_quant_type="nf4",
+    bnb_4bit_use_double_quant=True,
+    llm_int8_enable_fp32_cpu_offload=True # Crucial fallback for free CPU spaces
+)
+
+print("Loading compressed base model...")
 base_model = AutoModelForCausalLM.from_pretrained(
     BASE_MODEL,
-    torch_dtype=torch.float32,
-    device_map="cpu"
+    quantization_config=quantization_config,
+    device_map="auto"
 )
 
 print("Merging your custom fine-tuned engineering weights...")
-# This layers your specialized tasks right over the active model architecture
 model = PeftModel.from_pretrained(base_model, ADAPTER_MODEL)
 
 def chat_function(message, history):
@@ -54,7 +64,7 @@ def chat_function(message, history):
 demo = gr.ChatInterface(
     fn=chat_function,
     title="🤖 Cydercoder Qwen 3B AI Chatbot",
-    description="Your custom fine-tuned assistant running 24/7 in the cloud for free.",
+    description="Your custom fine-tuned assistant running compressed for speed in the cloud.",
     examples=["Write a login form using React and Tailwind.", "Fix this code error: Cannot read properties of undefined"]
 )