Update app.py

app.py

@@ -22,12 +22,16 @@ MODEL_CONFIG = {
     "lora_model": "programci48/heytak-lora-v1",
     "cache_dir": "/tmp/huggingface",
     "device": "cuda" if torch.cuda.is_available() else "cpu",
-    "torch_dtype": torch.float16 if torch.cuda.is_available() else torch.float32
+    "torch_dtype": torch.float16 if torch.cuda.is_available() else torch.float32,
+    "offload_folder": "/tmp/offload"  # Offload için yeni dizin
 }
 
 def load_models() -> Dict[str, Any]:
     """Modelleri yükleyen fonksiyon"""
     try:
+        # Offload dizini oluştur
+        os.makedirs(MODEL_CONFIG["offload_folder"], exist_ok=True)
+        
         logger.info("Tokenizer yükleniyor...")
         tokenizer = AutoTokenizer.from_pretrained(
             MODEL_CONFIG["base_model"],
@@ -39,10 +43,11 @@ def load_models() -> Dict[str, Any]:
         base_model = AutoModelForCausalLM.from_pretrained(
             MODEL_CONFIG["base_model"],
             torch_dtype=MODEL_CONFIG["torch_dtype"],
-            device_map="auto",
+            device_map="auto" if MODEL_CONFIG["device"] == "cuda" else None,
             token=HF_TOKEN,
             low_cpu_mem_usage=True,
-            cache_dir=MODEL_CONFIG["cache_dir"]
+            cache_dir=MODEL_CONFIG["cache_dir"],
+            offload_folder=MODEL_CONFIG["offload_folder"]
         )
 
         logger.info("LoRA adaptörü yükleniyor...")