Update app.py

app.py

@@ -8,6 +8,7 @@ from fastapi import FastAPI, HTTPException
 from pydantic import BaseModel
 from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
 from contextlib import asynccontextmanager
+import torch
 
 # === Ortam değişkenleri
 os.environ.setdefault("HF_HOME", "/app/.cache")
@@ -22,7 +23,7 @@ def log(message):
 def trim_history(messages, max_blocks=20):
     return messages[-max_blocks:]
 
-def enforce_token_budget(tokenizer, system_prompt, history_messages, user_input, total_ctx=4096, max_new_tokens=200):
+def enforce_token_budget(tokenizer, system_prompt, history_messages, user_input, total_ctx=4096, max_new_tokens=128):
     system_tokens = len(tokenizer(system_prompt)['input_ids'])
     user_tokens = len(tokenizer(user_input)['input_ids'])
     history_tokens = sum(len(tokenizer(m['content'])['input_ids']) for m in history_messages)
@@ -68,8 +69,8 @@ async def lifespan(app: FastAPI):
         tokenizer.pad_token = tokenizer.eos_token
 
         quant_config = BitsAndBytesConfig(
-            load_in_4bit=True,  # ✅ 4-bit quantization
-            llm_int4_threshold=6.0
+            load_in_4bit=True,
+            bnb_4bit_compute_dtype=torch.float16  # ✅ float16 hızlandırma
         )
 
         model = AutoModelForCausalLM.from_pretrained(
@@ -78,7 +79,7 @@ async def lifespan(app: FastAPI):
             quantization_config=quant_config
         )
 
-        log("✅ Model ve tokenizer başarıyla hazır (4-bit quantized, BitsAndBytesConfig).")
+        log("✅ Model ve tokenizer başarıyla hazır (4-bit quantized, float16 compute).")
         yield
 
     except Exception as e:
@@ -101,7 +102,7 @@ def generate(req: UserInputRequest):
         log(f"💬 Kullanıcı isteği alındı: {req.user_input}")
 
         trimmed_history = trim_history(req.history, max_blocks=20)
-        trimmed_history = enforce_token_budget(tokenizer, req.system_prompt, trimmed_history, req.user_input, total_ctx=4096, max_new_tokens=200)
+        trimmed_history = enforce_token_budget(tokenizer, req.system_prompt, trimmed_history, req.user_input, total_ctx=4096, max_new_tokens=128)
 
         # === Apply chat template
         t0 = time.time()
@@ -134,7 +135,7 @@ def generate(req: UserInputRequest):
 
         input_len = input_ids.shape[-1]
         total_ctx = model.config.max_position_embeddings if hasattr(model.config, 'max_position_embeddings') else 4096
-        max_new_tokens = min(200, max(1, total_ctx - input_len))
+        max_new_tokens = min(128, max(1, total_ctx - input_len))
 
         log(f"ℹ️ Input uzunluğu: {input_len}, max_new_tokens ayarlandı: {max_new_tokens}")