import sys
import os
# Add the project root directory to the Python path to resolve module imports
project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
sys.path.insert(0, project_root)
import os
import re
import datetime
import json
from retrieval_manager import RetrievalManager
import llm_interface
# --- Configuration ---
TEST_QUERIES_FILE = "./test_user_queries.md"
OUTPUT_HTML_FILE = "logs/chatbot_test_report_revised.html"
# --- HTML Template ---
HTML_TEMPLATE = """
Chatbot Test Report
Chatbot Test Report
Generated on: {generation_time}
{test_results}
"""
def parse_test_queries(file_path):
"""Reads and parses queries from the markdown file."""
queries = []
try:
with open(file_path, 'r') as f:
for line in f:
# Match lines that start with a number and a dot, and extract the quoted string
match = re.search(r'^\d+\.\s*"(.*)"', line)
if match:
queries.append(match.group(1))
except FileNotFoundError:
print(f"Error: Test queries file not found at '{file_path}'")
return []
return queries
def format_rag_content_as_html(rag_docs):
"""Formats the retrieved documents into a readable HTML string."""
if not rag_docs:
return "No documents were retrieved.
"
html = ""
for i, (content, metadata) in enumerate(rag_docs):
html += f"Document {i+1}
"
# Use json.dumps for pretty printing the metadata dictionary
metadata_str = json.dumps(metadata, indent=2)
html += f"Content:\n{content}\n\nMetadata:\n{metadata_str}
"
return html
def process_query(query, retriever):
"""Retrieves documents and generates a response for a single query."""
# 1. Retrieve documents using the same logic as the main app
search_results = retriever.search(query)
retrieved_docs = []
for collection_name, results in search_results.items():
if results and results.get('documents') and results['documents'][0]:
docs = results['documents'][0]
metadatas = results['metadatas'][0]
for i, doc_content in enumerate(docs):
retrieved_docs.append((doc_content, metadatas[i]))
# 2. Prepare context for LLM using the same logic as the main app
doc_contents = []
for content, metadata in retrieved_docs:
enhanced_content = content
if metadata:
metadata_parts = []
if 'product_name' in metadata and metadata['product_name'] not in enhanced_content:
metadata_parts.append(f"Product Name: {metadata['product_name']}")
if 'brand' in metadata and metadata['brand'] not in enhanced_content:
metadata_parts.append(f"Brand: {metadata['brand']}")
if 'category' in metadata and metadata['category'] not in enhanced_content:
metadata_parts.append(f"Category: {metadata['category']}")
if 'price' in metadata:
metadata_parts.append(f"Price: ${metadata['price']:.2f}")
if 'rating' in metadata:
metadata_parts.append(f"Rating: {metadata['rating']} out of 5")
if metadata_parts:
enhanced_content += "\n" + ", ".join(metadata_parts)
doc_contents.append(enhanced_content)
# 3. Generate response
# Using an empty chat history to ensure each test is isolated
response = llm_interface.generate_response(query, doc_contents, [])
return response, retrieved_docs
def main():
"""Main function to run the test script."""
print("Starting chatbot response test...")
# Ensure logs directory exists
if not os.path.exists("logs"):
os.makedirs("logs")
queries = parse_test_queries(TEST_QUERIES_FILE)
if not queries:
print("No queries found. Exiting.")
return
retriever = RetrievalManager()
all_results_html = ""
for i, query in enumerate(queries):
print(f"Processing query {i+1}/{len(queries)}: '{query}'")
response, rag_docs = process_query(query, retriever)
result_html = f'''
Query: "{query}"
► Show Retrieved Documents
{format_rag_content_as_html(rag_docs)}
{response}
'''
# Check if retry is needed
if "i'm sorry" in response.lower() or "i am sorry" in response.lower():
retry_query = f"try harder... {query}"
print(f" -> Retrying with: '{retry_query}'")
retry_response, retry_rag_docs = process_query(retry_query, retriever)
result_html += f'''
Retry Query: "{retry_query}"
► Show Retrieved Documents (Retry)
{format_rag_content_as_html(retry_rag_docs)}
{retry_response}
'''
result_html += "
"
all_results_html += result_html
# Final HTML content
final_html = HTML_TEMPLATE.format(
generation_time=datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
test_results=all_results_html
)
# Write to file
with open(OUTPUT_HTML_FILE, 'w', encoding='utf-8') as f:
f.write(final_html)
print(f"\nTest complete. Report saved to '{os.path.abspath(OUTPUT_HTML_FILE)}'")
if __name__ == "__main__":
main()