Upload app.py

app.py

@@ -66,6 +66,7 @@ def respond(
     temperature,
     top_p,
     embeddings_data,
+    tokenizer,
     model
 ):
     logging.info(f"New user query: {message}")
@@ -73,12 +74,12 @@ def respond(
     start_time = time.time()
 
     # Search for relevant documents based on user input
-    relevant_docs = get_relevant_documents(message, embeddings_data, model)
+    relevant_docs = get_relevant_documents(message, embeddings_data, tokenizer, model)
     retrieved_context = format_documents(relevant_docs)
     
     # Log the statistics about the retrieved documents
     logging.info(f"Total documents retrieved: {len(relevant_docs)}")
-    logging.info(f"Documents: " + {[doc['name'] for doc in relevant_docs]})
+    logging.info(f"Documents: " + str([doc['name'] for doc in relevant_docs]))
     
     # Add the retrieved context as part of the system message
     system_message_with_context = system_message + "\n\n" + "Relevant documents:\n" + retrieved_context
@@ -95,10 +96,12 @@ def respond(
     messages.append({"role": "user", "content": message})
     logging.info("Messages prepared for InferenceClient")
 
-    response = ""
     client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
     
     logging.info("Sending request to InferenceClient")
+    response = ""
+    
+    # Collect the full response instead of yielding each token
     for message in client.chat_completion(
         messages,
         max_tokens=max_tokens,
@@ -108,11 +111,13 @@ def respond(
     ):
         token = message.choices[0].delta.content
         response += token
-        yield response
     
     end_time = time.time()
     total_duration = end_time - start_time
     logging.info(f"Response generated in {total_duration:.2f} seconds")
+    
+    return response  # Return the complete response as a string
+
 
 # Load embeddings and model once at startup
 embeddings_file = 'Code Civil vectorised.json'