Update app.py

app.py

@@ -154,6 +154,11 @@ qwen3_tokenizer = AutoTokenizer.from_pretrained(qwen3_model_name)
 qwen3_model = AutoModelForCausalLM.from_pretrained(qwen3_model_name)
 qwen3_model = qwen3_model.to(device)
 
+qwen3_gguf_llm = Llama.from_pretrained(
+    repo_id="unsloth/Qwen3-0.6B-GGUF",
+    filename="Qwen3-0.6B-BF16.gguf",
+)
+
 
 # --- Generation Functions ---
 
@@ -243,11 +248,25 @@ def generate_qwen3(prompt: str) -> (str, str):
     else:
         return "", generated_text.strip()
 
+def generate_qwen3_gguf(prompt: str) -> (str, str):
+    messages = [
+        {"role": "user", "content": prompt}
+    ]
+    response = qwen3_gguf_llm.create_chat_completion(messages=messages)
+    generated_text = response['choices'][0]['message']['content']
+
+    if "</think>" in generated_text:
+        reasoning_content, content = generated_text.split("</think>", 1)
+        return reasoning_content.strip() + "</think>", content.strip()
+    else:
+        return "", generated_text.strip()
+
+
 
 @app.post("/generate/{model_name}", response_model=GenerateResponse)
 async def generate(
     request: PromptRequest,
-    model_name: str = Path(..., description="Model to use: 'deepseekr1-qwen', 't5-large', 'pegasus-large', or 'qwen3-0.6b'")
+    model_name: str = Path(..., description="Model to use: 'deepseekr1-qwen', 't5-large', 'pegasus-large', 'qwen3-0.6b-hf', or 'qwen3-0.6b-gguf'")
 ):
     if model_name == "deepseekr1-qwen":
         reasoning, text = generate_deepseek(request.prompt)
@@ -255,8 +274,10 @@ async def generate(
         reasoning, text = generate_t5(request.prompt)
     elif model_name == "pegasus-large":
         reasoning, text = generate_pegasus(request.prompt)
-    elif model_name == "qwen3-0.6b":
-        reasoning, text = generate_qwen3(request.prompt)
+    elif model_name == "qwen3-0.6b-hf":
+        reasoning, text = generate_qwen3_hf(request.prompt)
+    elif model_name == "qwen3-0.6b-gguf":
+        reasoning, text = generate_qwen3_gguf(request.prompt)
     else:
         return GenerateResponse(reasoning_content="", generated_text=f"Error: Unknown model '{model_name}'.")