ULTRA SPEED: 8-bit quantization, greedy decoding, 40 tokens, inference_mode

app.py

@@ -27,10 +27,10 @@ BASE_MODEL = "google/gemma-1.1-2b-it"
 print("Loading tokenizer...")
 tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL, trust_remote_code=True, token=HF_TOKEN)
 
-print("Loading base model...")
+print("Loading base model with 8-bit quantization for speed...")
 base_model = AutoModelForCausalLM.from_pretrained(
     BASE_MODEL,
-    torch_dtype=torch.float16,
+    load_in_8bit=True,  # 8-bit quantization for 2-3x speedup
     low_cpu_mem_usage=True,
     trust_remote_code=True,
     token=HF_TOKEN,
@@ -41,6 +41,13 @@ print("Loading LoRA adapters...")
 model = PeftModel.from_pretrained(base_model, MODEL_NAME, token=HF_TOKEN)
 model.eval()
 
+# Enable torch compile for faster inference (if available)
+try:
+    model = torch.compile(model, mode="reduce-overhead")
+    print("Torch compile enabled for faster inference")
+except Exception as e:
+    print(f"Torch compile not available: {e}")
+
 # Device'ı belirle
 device = "cuda" if torch.cuda.is_available() else "cpu"
 print(f"Using device: {device}")
@@ -129,18 +136,19 @@ def chat_completions():
         import time
         start_time = time.time()
         
-        outputs = model.generate(
-            **inputs,
-            max_new_tokens=80,  # Balanced: fast but complete (reduced from 150)
-            temperature=0.6,  # Slightly lower for faster convergence
-            do_sample=True,
-            top_p=0.85,  # Reduced for faster sampling
-            pad_token_id=tokenizer.pad_token_id,
-            eos_token_id=tokenizer.eos_token_id
-        )
+        # Ultra-fast generation settings for <1s response
+        with torch.inference_mode():  # Disable gradient computation for speed
+            outputs = model.generate(
+                **inputs,
+                max_new_tokens=40,  # Minimal tokens for ultra-fast response
+                do_sample=False,  # Greedy decoding (fastest)
+                pad_token_id=tokenizer.pad_token_id,
+                eos_token_id=tokenizer.eos_token_id,
+                use_cache=True  # Enable KV cache for faster generation
+            )
         
         elapsed = time.time() - start_time
-        print(f"Response generated in {elapsed:.2f}s")
+        print(f"⚡ Response generated in {elapsed:.2f}s")
 
         full_response = tokenizer.decode(outputs[0], skip_special_tokens=False)
         

requirements.txt

@@ -7,3 +7,4 @@ torch>=2.0.0
 accelerate>=0.25.0
 sentencepiece>=0.1.99
 protobuf>=3.20.0
+bitsandbytes>=0.41.0