""" Main Entry Point for Advanced RAG System Provides user-friendly interface for document upload and query processing """ import os import sys import json import time import argparse from pathlib import Path from typing import List, Dict, Any, Optional # Import our RAG system from rag_system import AdvancedRAGSystem, QueryResult # Import document processor for direct access from document_processer import AdvancedDocumentProcessor # Import pytesseract for direct OCR try: import pytesseract from PIL import Image import cv2 import numpy as np OCR_AVAILABLE = True except ImportError: OCR_AVAILABLE = False def print_banner(): """Print system banner""" print("=" * 80) print("πŸš€ ADVANCED RAG SYSTEM - INSURANCE POLICY ANALYZER") print("=" * 80) print("πŸ“‹ Features:") print(" β€’ Multi-format document processing (PDF, TXT, Email, etc.)") print(" β€’ Advanced OCR for scanned documents") print(" β€’ Natural language query processing") print(" β€’ Clause referencing and explanation") print(" β€’ Comprehensive audit trail") print(" β€’ GPU-accelerated processing") print("=" * 80) def print_menu(): """Print main menu options""" print("\nπŸ“‹ MAIN MENU:") print("1. πŸ“„ Upload Document") print("2. πŸ€” Process Query") print("3. πŸ“Š System Statistics") print("4. πŸ” View Audit Trail") print("5. πŸ§ͺ System Validation") print("6. πŸ’Ύ Export System Data") print("7. πŸ—‘οΈ Clear System") print("8. πŸ§ͺ Test OCR") print("9. ❓ Help") print("0. πŸšͺ Exit") print("-" * 40) def get_user_choice() -> str: """Get user choice from menu""" try: choice = input("\n🎯 Enter your choice (1-9): ").strip() return choice except KeyboardInterrupt: print("\nπŸ‘‹ Goodbye!") sys.exit(0) def simple_ocr_pdf(file_path: str) -> str: """Simple OCR function using pytesseract directly""" if not OCR_AVAILABLE: print("❌ pytesseract not available. Please install it with: pip install pytesseract") return "" try: import fitz # PyMuPDF # Open PDF doc = fitz.open(file_path) all_text = "" for page_num in range(len(doc)): page = doc.load_page(page_num) # Get page as image pix = page.get_pixmap() img = Image.frombytes("RGB", [pix.width, pix.height], pix.samples) # Extract text using OCR text = pytesseract.image_to_string(img, lang='eng', config='--psm 6') all_text += f"\n\n--- Page {page_num + 1} ---\n{text}" doc.close() return all_text except Exception as e: print(f"❌ OCR error: {e}") return "" def upload_document(rag_system: AdvancedRAGSystem): """Handle document upload""" print("\nπŸ“„ DOCUMENT UPLOAD") print("-" * 40) try: # Get file path file_path = input("πŸ“ Enter file path (or 'browse' for file dialog): ").strip() if file_path.lower() == 'browse': try: import tkinter as tk from tkinter import filedialog root = tk.Tk() root.withdraw() file_path = filedialog.askopenfilename( title="Select Document", filetypes=[ ("All supported", "*.pdf;*.txt;*.docx;*.html;*.eml;*.csv;*.json"), ("PDF files", "*.pdf"), ("Text files", "*.txt"), ("Word documents", "*.docx"), ("HTML files", "*.html"), ("Email files", "*.eml"), ("CSV files", "*.csv"), ("JSON files", "*.json"), ("All files", "*.*") ] ) root.destroy() if not file_path: print("❌ No file selected.") return except ImportError: print("❌ File browser not available. Please enter the file path manually.") return except Exception as e: print(f"❌ Error opening file browser: {e}") return # Check if file exists if not os.path.exists(file_path): print(f"❌ File not found: {file_path}") return # Check file type supported_extensions = {'.pdf', '.txt', '.docx', '.html', '.htm', '.eml', '.msg', '.csv', '.json'} file_extension = Path(file_path).suffix.lower() if file_extension not in supported_extensions: print(f"❌ Unsupported file type: {file_extension}") print(f"πŸ’‘ Supported types: {', '.join(supported_extensions)}") return # Ask about OCR for PDF files use_ocr = False if file_extension == '.pdf': if OCR_AVAILABLE: ocr_choice = input("πŸ” Use OCR for scanned documents? (y/n, default: n): ").strip().lower() use_ocr = ocr_choice in ['y', 'yes'] else: print("⚠️ OCR not available. Processing without OCR.") print(f"\n⏳ Processing document: {os.path.basename(file_path)}") print("πŸ’‘ This may take a few moments...") # Process document start_time = time.time() try: chunks = rag_system.ingest_document(file_path, use_ocr=use_ocr) processing_time = time.time() - start_time except Exception as e: print(f"❌ Document ingestion failed: {e}") print("πŸ’‘ Trying alternative processing method...") # Fallback: Use document processor directly try: doc_processor = AdvancedDocumentProcessor() chunks = doc_processor.process_document(file_path, use_ocr=use_ocr) # Add chunks to vector database manually if chunks: success = rag_system.vector_database.add_documents(chunks) if success: print("βœ… Document processed using fallback method") processing_time = time.time() - start_time else: print("❌ Failed to add documents to vector database") chunks = [] else: print("❌ No chunks generated from document") chunks = [] processing_time = time.time() - start_time except Exception as fallback_error: print(f"❌ Fallback processing also failed: {fallback_error}") chunks = [] processing_time = time.time() - start_time if chunks: print(f"βœ… Successfully processed {len(chunks)} chunks in {processing_time:.2f} seconds") print(f"πŸ“Š Document chunks created:") for i, chunk in enumerate(chunks[:5]): # Show first 5 chunks print(f" Chunk {i+1}: {chunk.chunk_id}") print(f" Type: {chunk.section_type}") print(f" Content preview: {chunk.content[:100]}...") print() if len(chunks) > 5: print(f" ... and {len(chunks) - 5} more chunks") else: print("❌ Failed to process document") if file_extension == '.pdf' and not use_ocr: print("πŸ’‘ Try using OCR for scanned PDFs: Select 'y' when asked about OCR") except KeyboardInterrupt: print("\nπŸ‘‹ Upload cancelled.") except Exception as e: print(f"❌ Error uploading document: {e}") def process_query(rag_system: AdvancedRAGSystem): """Interactive query processing with continuous questioning""" print("\nπŸ€” INTERACTIVE QUERY PROCESSING") print("-" * 40) print("πŸ’‘ Enter your questions about the uploaded documents.") print("πŸ’‘ Type 'quit' or 'exit' to stop asking questions.") print("πŸ’‘ Type 'help' for example questions.") print() results = [] query_count = 0 while True: try: # Get user input user_query = input("πŸ€” Enter your question: ").strip() # Check for exit commands if user_query.lower() in ['quit', 'exit', 'q']: print("πŸ‘‹ Goodbye!") break # Check for help command if user_query.lower() == 'help': print("\nπŸ“‹ Example questions you can ask:") print(" β€’ Is heart surgery covered under this policy?") print(" β€’ What is the waiting period for pre-existing diseases?") print(" β€’ Can I claim for dental treatment?") print(" β€’ What is the maximum coverage amount?") print(" β€’ Are there any exclusions for chronic diseases?") print(" β€’ What documents are required for claim submission?") print(" β€’ Is cancer treatment covered?") print(" β€’ What is the claim process?") print(" β€’ Does the policy cover newborn care after hospital discharge?") print() continue # Skip empty queries if not user_query: print("⚠️ Please enter a question.") continue query_count += 1 print(f"\nπŸ” Processing Query #{query_count}") print(f"πŸ€” Query: {user_query}") # Process the query start_time = time.time() try: result = rag_system.process_query(user_query) processing_time = time.time() - start_time except Exception as e: print(f"❌ Query processing failed: {e}") print("πŸ’‘ Creating fallback result...") # Create fallback result from query_parser import AdvancedQueryParser from llm_reasoning import AdvancedLLMReasoning try: # Parse query query_parser = AdvancedQueryParser() parsed_query = query_parser.parse_query(user_query) # Create fallback reasoning reasoning_engine = AdvancedLLMReasoning() fallback_context = [{ 'content': f"Based on the query: {user_query}", 'source_file': 'fallback', 'similarity_score': 0.5 }] reasoning_result = reasoning_engine.analyze_query( user_query, fallback_context, parsed_query.query_type ) # Create fallback result from rag_system import QueryResult result = QueryResult( query=user_query, parsed_query=parsed_query, search_results=[], reasoning_result=reasoning_result, processing_time=time.time() - start_time, timestamp=time.time(), audit_trail={'status': 'fallback', 'error': str(e)} ) processing_time = time.time() - start_time except Exception as fallback_error: print(f"❌ Fallback processing also failed: {fallback_error}") continue # Display results print(f"\nπŸ“‹ QUERY RESULTS") print("=" * 50) print(f"⏱️ Processing time: {processing_time:.2f} seconds") print(f"🎯 Decision: {result.reasoning_result.decision.upper()}") print(f"πŸ“Š Confidence: {result.reasoning_result.confidence_score:.1%}") if result.reasoning_result.amount: print(f"πŸ’° Amount: ${result.reasoning_result.amount:,.2f}") if result.reasoning_result.waiting_period: print(f"⏰ Waiting Period: {result.reasoning_result.waiting_period}") print(f"\nπŸ“ Justification:") print(result.reasoning_result.justification) if result.reasoning_result.relevant_clauses: print(f"\nπŸ“„ Relevant Clauses:") for clause in result.reasoning_result.relevant_clauses: print(f" β€’ {clause}") if result.reasoning_result.conditions: print(f"\nβœ… Conditions:") for condition in result.reasoning_result.conditions: print(f" β€’ {condition}") if result.reasoning_result.exclusions: print(f"\n❌ Exclusions:") for exclusion in result.reasoning_result.exclusions: print(f" β€’ {exclusion}") if result.reasoning_result.required_documents: print(f"\nπŸ“‹ Required Documents:") for doc in result.reasoning_result.required_documents: print(f" β€’ {doc}") # Show search results summary if result.search_results: print(f"\nπŸ” Search Results Summary:") print(f" Found {len(result.search_results)} relevant documents") for i, search_result in enumerate(result.search_results[:3], 1): print(f" {i}. {search_result.source_file} (Score: {search_result.similarity_score:.2f})") # Ask if user wants to see detailed explanation show_details = input("\n❓ Show detailed explanation? (y/n): ").strip().lower() if show_details in ['y', 'yes']: try: detailed_explanation = rag_system.reasoning_engine.explain_decision(result.reasoning_result) print(f"\nπŸ“– DETAILED EXPLANATION:") print("=" * 50) print(detailed_explanation) except Exception as e: print(f"❌ Error generating detailed explanation: {e}") print("πŸ’‘ Detailed explanation not available") results.append({ "query": user_query, "result": result, "processing_time": processing_time }) print() # Ask if user wants to continue if query_count % 3 == 0: # Ask every 3 queries continue_choice = input("❓ Continue asking questions? (y/n): ").strip().lower() if continue_choice not in ['y', 'yes', '']: print("πŸ‘‹ Thanks for using the RAG system!") break except KeyboardInterrupt: print("\nπŸ‘‹ Interrupted by user. Goodbye!") break except Exception as e: print(f"❌ Error processing query: {e}") print("πŸ’‘ Try asking a different question or type 'help' for examples.") print() return results def show_system_statistics(rag_system: AdvancedRAGSystem): """Show system statistics""" print("\nπŸ“Š SYSTEM STATISTICS") print("-" * 40) try: stats = rag_system.get_system_statistics() if stats: # Vector database stats db_stats = stats.get('vector_database', {}) print(f"πŸ“š Vector Database:") print(f" Total chunks: {db_stats.get('total_chunks', 0)}") print(f" Unique sources: {db_stats.get('unique_sources', 0)}") print(f" File types: {', '.join(db_stats.get('file_types', []))}") # Audit trail stats audit_stats = stats.get('audit_trail', {}) print(f"\nπŸ“‹ Audit Trail:") print(f" Total entries: {audit_stats.get('total_entries', 0)}") print(f" Successful queries: {audit_stats.get('successful_queries', 0)}") print(f" Failed queries: {audit_stats.get('failed_queries', 0)}") print(f" Document ingestions: {audit_stats.get('document_ingestions', 0)}") # Component info components = stats.get('components', {}) print(f"\nπŸ”§ Components:") print(f" Document Processor: {components.get('document_processor', 'Unknown')}") print(f" Vector Database: {components.get('vector_database', 'Unknown')}") print(f" Query Parser: {components.get('query_parser', 'Unknown')}") print(f" Reasoning Engine: {components.get('reasoning_engine', 'Unknown')}") print(f" GPU Enabled: {components.get('use_gpu', False)}") else: print("❌ Unable to retrieve system statistics") except Exception as e: print(f"❌ Error getting system statistics: {e}") def show_audit_trail(rag_system: AdvancedRAGSystem): """Show audit trail""" print("\nπŸ” AUDIT TRAIL") print("-" * 40) try: audit_log = rag_system.get_audit_trail() if audit_log: print(f"πŸ“‹ Total entries: {len(audit_log)}") # Show recent entries recent_entries = audit_log[-10:] # Last 10 entries print(f"\nπŸ“ Recent Entries:") for i, entry in enumerate(reversed(recent_entries), 1): action = entry.get('action', 'Unknown') timestamp = entry.get('timestamp', 'Unknown') status = entry.get('status', 'Unknown') print(f" {i}. {action} - {status} ({timestamp})") if action == 'query_processing': query = entry.get('query', 'Unknown') print(f" Query: {query[:50]}...") reasoning = entry.get('reasoning_result', {}) if reasoning: decision = reasoning.get('decision', 'Unknown') confidence = reasoning.get('confidence_score', 0.0) print(f" Decision: {decision} (Confidence: {confidence:.1%})") elif action == 'document_ingestion': file_path = entry.get('file_path', 'Unknown') chunks = entry.get('chunks_processed', 0) print(f" File: {os.path.basename(file_path)} ({chunks} chunks)") print() # Ask if user wants to save audit trail save_choice = input("πŸ’Ύ Save audit trail to file? (y/n): ").strip().lower() if save_choice in ['y', 'yes']: filename = input("πŸ“ Enter filename (default: audit_trail.json): ").strip() if not filename: filename = "audit_trail.json" if rag_system.save_audit_trail(filename): print(f"βœ… Audit trail saved to: {filename}") else: print("❌ Failed to save audit trail") else: print("πŸ“‹ No audit trail entries found") except Exception as e: print(f"❌ Error showing audit trail: {e}") def validate_system(rag_system: AdvancedRAGSystem): """Validate system components""" print("\nπŸ§ͺ SYSTEM VALIDATION") print("-" * 40) try: validation = rag_system.validate_system() print("πŸ” Checking system components...") components = [ ('Document Processor', validation.get('document_processor', False)), ('Vector Database', validation.get('vector_database', False)), ('Query Parser', validation.get('query_parser', False)), ('Reasoning Engine', validation.get('reasoning_engine', False)) ] all_valid = True for component_name, is_valid in components: status = "βœ… PASS" if is_valid else "❌ FAIL" print(f" {component_name}: {status}") if not is_valid: all_valid = False print(f"\n🎯 Overall Status: {'βœ… ALL COMPONENTS VALID' if all_valid else '❌ SOME COMPONENTS FAILED'}") if not all_valid: print("\n❌ Errors found:") for error in validation.get('errors', []): print(f" β€’ {error}") except Exception as e: print(f"❌ Error validating system: {e}") def export_system_data(rag_system: AdvancedRAGSystem): """Export system data""" print("\nπŸ’Ύ EXPORT SYSTEM DATA") print("-" * 40) try: filename = input("πŸ“ Enter filename (default: system_export.json): ").strip() if not filename: filename = "system_export.json" print(f"⏳ Exporting system data to: {filename}") if rag_system.export_system_data(filename): print(f"βœ… System data exported successfully to: {filename}") # Show file size if os.path.exists(filename): file_size = os.path.getsize(filename) print(f"πŸ“ File size: {file_size:,} bytes") else: print("❌ Failed to export system data") except Exception as e: print(f"❌ Error exporting system data: {e}") def clear_system(rag_system: AdvancedRAGSystem): """Clear system data""" print("\nπŸ—‘οΈ CLEAR SYSTEM") print("-" * 40) try: confirm = input("⚠️ This will clear ALL system data. Are you sure? (yes/no): ").strip().lower() if confirm == 'yes': print("⏳ Clearing system data...") if rag_system.clear_system(): print("βœ… System data cleared successfully") else: print("❌ Failed to clear system data") else: print("❌ Operation cancelled") except Exception as e: print(f"❌ Error clearing system: {e}") def test_ocr(): """Test OCR functionality""" print("\nπŸ§ͺ OCR TEST") print("-" * 40) if not OCR_AVAILABLE: print("❌ pytesseract not available") print("πŸ’‘ Install with: pip install pytesseract") return try: # Create a simple test image from PIL import Image, ImageDraw, ImageFont # Create test image img = Image.new('RGB', (300, 100), color='white') draw = ImageDraw.Draw(img) # Try to use a default font try: font = ImageFont.load_default() except: font = None # Draw text text = "Test OCR Text" draw.text((10, 40), text, fill='black', font=font) # Test OCR ocr_text = pytesseract.image_to_string(img) print(f"βœ… OCR test successful") print(f" Original: '{text}'") print(f" OCR result: '{ocr_text.strip()}'") except Exception as e: print(f"❌ OCR test failed: {e}") print("πŸ’‘ Make sure Tesseract is installed and configured") def show_help(): """Show help information""" print("\n❓ HELP") print("-" * 40) print("πŸ“‹ This RAG system can process various document types and answer questions about them.") print("\nπŸ“„ Supported Document Types:") print(" β€’ PDF files (with OCR support for scanned documents)") print(" β€’ Text files (.txt)") print(" β€’ Word documents (.docx)") print(" β€’ HTML files (.html)") print(" β€’ Email files (.eml, .msg)") print(" β€’ CSV files (.csv)") print(" β€’ JSON files (.json)") print("\nπŸ€” Query Examples:") print(" β€’ 'Is heart surgery covered under this policy?'") print(" β€’ 'How do I file a claim?'") print(" β€’ 'What is the waiting period for pre-existing conditions?'") print(" β€’ 'What documents are required for claim submission?'") print(" β€’ 'What is the maximum coverage amount?'") print("\nπŸ’‘ Tips:") print(" β€’ Upload documents first before asking questions") print(" β€’ Use natural language - the system understands plain English") print(" β€’ The system can handle vague or incomplete queries") print(" β€’ All queries are logged in the audit trail") print(" β€’ Use the system validation to check component status") print(" β€’ Test OCR functionality if you have issues with scanned documents") def main(): """Main function""" # Parse command line arguments parser = argparse.ArgumentParser(description='Advanced RAG System') parser.add_argument('--query', help='Process a single query from file') parser.add_argument('--upload', help='Upload and process a document') parser.add_argument('--status', action='store_true', help='Check system status') args = parser.parse_args() # Handle command line arguments if args.query: # Process single query from file try: with open(args.query, 'r') as f: question = f.read().strip() # Initialize RAG system rag_system = AdvancedRAGSystem(use_gpu=False) # Process the query result = rag_system.process_query(question) # Output results in a structured format print(f"Question: {question}") print(f"Decision: {result.reasoning_result.decision}") print(f"Confidence: {result.reasoning_result.confidence_score:.1%}") print(f"Justification: {result.reasoning_result.justification}") if result.reasoning_result.amount: print(f"Amount: ${result.reasoning_result.amount:,.2f}") if result.reasoning_result.waiting_period: print(f"Waiting Period: {result.reasoning_result.waiting_period}") if result.reasoning_result.relevant_clauses: print(f"Relevant Clauses: {', '.join(result.reasoning_result.relevant_clauses)}") if result.reasoning_result.conditions: print(f"Conditions: {', '.join(result.reasoning_result.conditions)}") if result.reasoning_result.exclusions: print(f"Exclusions: {', '.join(result.reasoning_result.exclusions)}") if result.reasoning_result.required_documents: print(f"Required Documents: {', '.join(result.reasoning_result.required_documents)}") return except Exception as e: print(f"Error processing query: {e}") sys.exit(1) elif args.upload: # Upload and process document try: # Initialize RAG system rag_system = AdvancedRAGSystem(use_gpu=False) # Process the document chunks = rag_system.ingest_document(args.upload, use_ocr=False) print(f"Document processed successfully: {args.upload}") print(f"Chunks processed: {len(chunks)}") return except Exception as e: print(f"Error processing document: {e}") sys.exit(1) elif args.status: # Check system status try: rag_system = AdvancedRAGSystem(use_gpu=False) print("System Status: READY") return except Exception as e: print(f"System Status: ERROR - {e}") sys.exit(1) # Interactive mode (default) print_banner() try: # Initialize RAG system print("πŸš€ Initializing RAG system...") try: rag_system = AdvancedRAGSystem(use_gpu=False) # Use CPU for better compatibility print("βœ… RAG system initialized successfully!") except Exception as e: print(f"❌ Failed to initialize RAG system: {e}") print("πŸ’‘ Trying with minimal configuration...") try: # Try with minimal settings rag_system = AdvancedRAGSystem( use_gpu=False, model_path=None # Don't require model file ) print("βœ… RAG system initialized with minimal configuration!") except Exception as e2: print(f"❌ RAG system initialization failed: {e2}") print("πŸ’‘ Please check your system configuration") return # Main loop while True: print_menu() choice = get_user_choice() if choice == '1': upload_document(rag_system) elif choice == '2': process_query(rag_system) elif choice == '3': show_system_statistics(rag_system) elif choice == '4': show_audit_trail(rag_system) elif choice == '5': validate_system(rag_system) elif choice == '6': export_system_data(rag_system) elif choice == '7': clear_system(rag_system) elif choice == '8': test_ocr() elif choice == '9': show_help() elif choice == '0': print("πŸ‘‹ Thank you for using the Advanced RAG System!") break else: print("❌ Invalid choice. Please enter a number between 1-0.") # Pause before showing menu again input("\n⏸️ Press Enter to continue...") except KeyboardInterrupt: print("\nπŸ‘‹ Goodbye!") except Exception as e: print(f"❌ Fatal error: {e}") print("πŸ’‘ Please check your system configuration and try again.") if __name__ == "__main__": main()