Update app.py

app.py

@@ -69,32 +69,46 @@ client = InferenceClient("google/gemma-3-27b-it")
     #    messages.extend(history)
         
    # messages.append({"role": "user", "content": message})
-   # response = ""
-   # for messages in client.chat_completion(messages,max_tokens = 2500, stream = True):
-    #    token = messages.choices[0].delta.content
-     #   response += token
-      #  yield response
+    #response = ""
+    #for messages in client.chat_completion(messages,max_tokens = 2500, stream = True):
+     #   token = messages.choices[0].delta.content
+      #  response += token
+       # yield response
 
 def respond(message, history):
-    # Get relevant information from knowledge base
-    relevant_chunks = get_top_chunks(message)
-    context = "\n".join(relevant_chunks)
-
-    # Build prompt
-    prompt = (
-        "You are Scooby, a helpful chatbot that only answers questions about pets and their diets.\n"
-        "Use the following relevant information to help answer the user's question:\n\n"
-        f"{context}\n\n"
-        f"User: {message}\nScooby:"
+    # Step 1: Embed the user's question
+    message_embedding = model.encode(message, convert_to_tensor=True)
+
+    # Step 2: Calculate similarity with knowledge chunks
+    scores = util.cos_sim(message_embedding, chunk_embeddings)[0]
+    top_k = 3  # You can adjust how many chunks you want to include
+    top_results = torch.topk(scores, k=top_k)
+
+    # Step 3: Retrieve the top relevant knowledge chunks
+    retrieved_knowledge = "\n".join([chunks[i] for i in top_results.indices])
+
+    # Step 4: Build system message with retrieved knowledge
+    system_message = (
+        "You are a helpful chatbot named Scooby, kind of like the cartoon character but not too much. "
+        "You know a lot about pets and their diets, and you only answer questions about pets. "
+        "Use the following relevant knowledge to help answer the user's question:\n\n"
+        + retrieved_knowledge
     )
 
-    # Stream response
-    response = ""
-    for chunk in client.text_generation(prompt=prompt, max_new_tokens=300, stream=True):
-        response += chunk.token.text
-        yield response
+    # Step 5: Compose message list for the LLM
+    messages = [{"role": "system", "content": system_message}]
+
+    if history:
+        messages.extend(history)
 
+    messages.append({"role": "user", "content": message})
 
+    # Step 6: Stream response
+    response = ""
+    for chunk in client.chat_completion(messages, max_tokens=2500, stream=True):
+        token = chunk.choices[0].delta.content
+        response += token
+        yield response
 
 #theme = gr.themes.Origin(primary_hue="orange",secondary_hue="indigo", neutral_hue="teal")