Update app.py

app.py

@@ -1,12 +1,13 @@
 from fastapi import FastAPI, HTTPException, Depends, status
 from fastapi.security import HTTPBearer
 from pydantic import BaseModel
-from transformers import pipeline
+from llama_cpp import Llama
 import gradio as gr
 import os
 from dotenv import load_dotenv
 import uvicorn
 import threading
+from huggingface_hub import snapshot_download
 
 load_dotenv()
 
@@ -17,7 +18,20 @@ API_KEY = os.getenv("API_KEY")
 if not API_KEY:
     raise ValueError("API_KEY not found in environment variables")
 
-pipe = pipeline("text-generation", model="unsloth/gemma-3-270m-it-GGUF", model_kwargs={"gguf_file": "gemma-3-270m-it-UD-Q8_K_XL.gguf"})
+os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
+
+snapshot_download(
+    repo_id="unsloth/gemma-3-270m-it-GGUF",
+    local_dir="gemma-3-270m-it-GGUF",
+    allow_patterns=["*UD-Q8_K_XL*"]
+)
+
+llm = Llama(
+    model_path="gemma-3-270m-it-GGUF/gemma-3-270m-it-UD-Q8_K_XL.gguf",
+    n_ctx=4096,
+    n_threads=2,
+    n_gpu_layers=0
+)
 
 def load_system_prompt():
     try:
@@ -44,21 +58,25 @@ def verify_api_key(credentials = Depends(security)):
 
 @app.post("/enhance", response_model=EnhanceResponse)
 async def enhance_prompt(request: EnhanceRequest, api_key: str = Depends(verify_api_key)):
-    messages = [
-        {"role": "system", "content": SYSTEM_PROMPT},
-        {"role": "user", "content": request.prompt}
-    ]
+    full_prompt = f"<start_of_turn>user\n{SYSTEM_PROMPT}\n\n{request.prompt}<end_of_turn>\n<start_of_turn>model\n"
     
     try:
-        result = pipe(messages, max_new_tokens=256, temperature=0.7, do_sample=True)
-        enhanced_prompt = result[0]["generated_text"]
+        result = llm(
+            full_prompt,
+            max_tokens=512,
+            temperature=0.7,
+            top_k=40,
+            top_p=0.95,
+            repeat_penalty=1.1,
+            stop=["<end_of_turn>"]
+        )
         
-        if isinstance(enhanced_prompt, list):
-            user_message = next((msg["content"] for msg in enhanced_prompt if msg["role"] == "assistant"), enhanced_prompt[-1]["content"])
-        else:
-            user_message = enhanced_prompt.split("assistant")[-1].strip() if "assistant" in enhanced_prompt else enhanced_prompt
+        enhanced_prompt = result["choices"][0]["text"].strip()
         
-        return EnhanceResponse(enhanced_prompt=user_message)
+        if not enhanced_prompt:
+            raise HTTPException(status_code=500, detail="Enhancement failed: Empty response")
+            
+        return EnhanceResponse(enhanced_prompt=enhanced_prompt)
     except Exception as e:
         raise HTTPException(status_code=500, detail=f"Enhancement failed: {str(e)}")
 
@@ -72,41 +90,25 @@ def enhance_for_gradio(prompt_text, api_key):
     if api_key != API_KEY:
         return "Invalid API key."
     
-    # Gemma model formatı
-    full_prompt = f"""<start_of_turn>user
-{SYSTEM_PROMPT}
-
-{prompt_text}<end_of_turn>
-<start_of_turn>model
-"""
+    full_prompt = f"<start_of_turn>user\n{SYSTEM_PROMPT}\n\n{prompt_text}<end_of_turn>\n<start_of_turn>model\n"
     
     try:
-        result = pipe(
+        result = llm(
             full_prompt,
-            max_new_tokens=1024,
+            max_tokens=512,
             temperature=0.7,
-            do_sample=True,
-            stop_strings=["<end_of_turn>"]
+            top_k=40,
+            top_p=0.95,
+            repeat_penalty=1.1,
+            stop=["<end_of_turn>"]
         )
         
-        response = result[0]["generated_text"]
-        print("=== RAW OUTPUT ===")
-        print(response)
-        print("=== END ===")
-        
-        # Yanıtı temizle
-        if full_prompt in response:
-            enhanced_prompt = response[len(full_prompt):].strip()
-        else:
-            enhanced_prompt = response.replace(full_prompt, "").strip()
-        
-        enhanced_prompt = enhanced_prompt.replace("<end_of_turn>", "").strip()
+        enhanced_prompt = result["choices"][0]["text"].strip()
         
         if not enhanced_prompt:
             return "Model generated empty response."
             
-        return enhanced_prompt[:2000]  # Çok uzunsa kısalt
-
+        return enhanced_prompt
     except Exception as e:
         return f"Enhancement failed: {str(e)}"