Update app.py

app.py

@@ -1,39 +1,55 @@
+import os
 import gradio as gr
 from qdrant_client import QdrantClient
 from sentence_transformers import SentenceTransformer
-from transformers import AutoTokenizer, AutoModelForCausalLM
-import torch
-import os
+from llama_cpp import Llama
+
 
 QDRANT_URL = os.environ.get("QDRANT_URL")
 QDRANT_API_KEY = os.environ.get("QDRANT_API_KEY")
 COLLECTION_NAME = "well_vectors"
 
-QWEN_MODEL = "Qwen/Qwen2.5-0.5B-Instruct"
+MODEL_PATH = "/model.gguf"   
 
 embedder = SentenceTransformer("all-MiniLM-L6-v2")
 
+
 client = QdrantClient(
     url=QDRANT_URL,
     api_key=QDRANT_API_KEY
 )
 
-tokenizer = AutoTokenizer.from_pretrained(QWEN_MODEL)
-model = AutoModelForCausalLM.from_pretrained(
-    QWEN_MODEL,
-    torch_dtype=torch.float16,
-    device_map="auto"
+llm = Llama(
+    model_path=MODEL_PATH,
+    n_ctx=4096,
+    n_threads=2,
+    n_batch=128,
+    verbose=False
 )
 
-def scientific_query_api(question: str):
 
+SYSTEM_PROMPT = """You are a scientific formatter.
+
+Rules:
+- You may ONLY use the provided information.
+- Do NOT add facts, examples, or interpretations.
+- Do NOT speculate.
+- Preserve scientific meaning exactly.
+- If information is insufficient, say so explicitly.
+"""
+
+def scientific_query_api(question: str):
+    # 1. Embed query
     qvec = embedder.encode(question, normalize_embeddings=True)
 
+    # 2. Concept retrieval
     concepts = client.search(
         collection_name=COLLECTION_NAME,
         query_vector=qvec,
         filter={
-            "must": [{"key": "type", "match": {"value": "concept"}}]
+            "must": [
+                {"key": "type", "match": {"value": "concept"}}
+            ]
         },
         limit=1
     )
@@ -48,12 +64,14 @@ def scientific_query_api(question: str):
 
     concept = concepts[0]
 
+
     evidence = client.search(
         collection_name=COLLECTION_NAME,
         query_vector=concept.vector,
         limit=5
     )
 
+
     packet = []
     packet.append("Concept definition:")
     packet.append(concept.payload["content"])
@@ -64,36 +82,35 @@ def scientific_query_api(question: str):
     for e in evidence:
         if "dataset" in e.payload:
             packet.append(
-                f"- Dataset: {e.payload['dataset']}, File: {e.payload.get('file','')}"
+                f"- Dataset: {e.payload['dataset']}, "
+                f"File: {e.payload.get('file', '')}"
             )
             sources.add(f"The Well: {e.payload['dataset']}")
 
     evidence_text = "\n".join(packet)
 
-    prompt = f"""
-You are a scientific formatter.
-
-Rules:
-- Use ONLY the information below.
-- Do NOT add facts or interpretations.
-- Preserve scientific meaning.
+    prompt = f"""{SYSTEM_PROMPT}
 
 INFORMATION:
 {evidence_text}
+
+Formatted explanation:
 """
 
-    inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
-    outputs = model.generate(
-        **inputs,
-        max_new_tokens=300,
-        do_sample=False
+    output = llm(
+        prompt,
+        max_tokens=300,
+        temperature=0.2,
+        top_p=0.9,
+        repeat_penalty=1.1,
+        stop=["INFORMATION:", "Formatted explanation:"]
     )
 
-    answer = tokenizer.decode(outputs[0], skip_special_tokens=True)
+    answer = output["choices"][0]["text"].strip()
 
     return {
         "question": question,
-        "answer": answer.strip(),
+        "answer": answer,
         "sources": sorted(sources),
         "confidence": "grounded"
     }
@@ -101,8 +118,7 @@ INFORMATION:
 iface = gr.Interface(
     fn=scientific_query_api,
     inputs=gr.Textbox(label="Scientific Question"),
-    outputs="json",
-    allow_flagging="never"
+    outputs="json"
 )
 
 iface.launch(server_name="0.0.0.0", server_port=7860)