Update app.py

app.py

@@ -10,21 +10,20 @@ LORA_PATH = "./"
 tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL)
 tokenizer.pad_token = tokenizer.eos_token
 
-# Base model 
+# Base model (disk offload enabled)
 base_model = AutoModelForCausalLM.from_pretrained(
-    repo_id,
-    device_map='auto',
-    offload_folder="offload/" )
+    BASE_MODEL,
+    torch_dtype=torch.float32,
+    device_map="auto",
+    offload_folder="offload",     
+    offload_state_dict=True,
+    low_cpu_mem_usage=True
+)
 
 base_model.config.use_cache = False
 
-model = PeftModel.from_pretrained(
-    base_model,
-    output_dir,
-    offload_folder = "offload/"
-    )
-# Load LoRA
-model = PeftModel.from_pretrained(model, LORA_PATH)
+# Load LoRA (SADECE 1 KEZ)
+model = PeftModel.from_pretrained(base_model, LORA_PATH)
 model.eval()
 
 
@@ -46,7 +45,7 @@ Answer:
         output = model.generate(
             **inputs,
             max_new_tokens=int(max_tokens),
-            do_sample=False,          
+            do_sample=False,
             eos_token_id=tokenizer.eos_token_id
         )
 
@@ -54,7 +53,6 @@ Answer:
     return tokenizer.decode(generated, skip_special_tokens=True)
 
 
-# Gradio UI
 demo = gr.Interface(
     fn=chat,
     inputs=[
@@ -67,4 +65,4 @@ demo = gr.Interface(
 )
 
 if __name__ == "__main__":
-    demo.launch(share=True)
+    demo.launch()