File size: 30,847 Bytes
09281fe
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
"""
Main Entry Point for Advanced RAG System
Provides user-friendly interface for document upload and query processing
"""

import os
import sys
import json
import time
import argparse
from pathlib import Path
from typing import List, Dict, Any, Optional

# Import our RAG system
from rag_system import AdvancedRAGSystem, QueryResult

# Import document processor for direct access
from document_processer import AdvancedDocumentProcessor

# Import pytesseract for direct OCR
try:
    import pytesseract
    from PIL import Image
    import cv2
    import numpy as np
    OCR_AVAILABLE = True
except ImportError:
    OCR_AVAILABLE = False

def print_banner():
    """Print system banner"""
    print("=" * 80)
    print("πŸš€ ADVANCED RAG SYSTEM - INSURANCE POLICY ANALYZER")
    print("=" * 80)
    print("πŸ“‹ Features:")
    print("   β€’ Multi-format document processing (PDF, TXT, Email, etc.)")
    print("   β€’ Advanced OCR for scanned documents")
    print("   β€’ Natural language query processing")
    print("   β€’ Clause referencing and explanation")
    print("   β€’ Comprehensive audit trail")
    print("   β€’ GPU-accelerated processing")
    print("=" * 80)

def print_menu():
    """Print main menu options"""
    print("\nπŸ“‹ MAIN MENU:")
    print("1. πŸ“„ Upload Document")
    print("2. πŸ€” Process Query")
    print("3. πŸ“Š System Statistics")
    print("4. πŸ” View Audit Trail")
    print("5. πŸ§ͺ System Validation")
    print("6. πŸ’Ύ Export System Data")
    print("7. πŸ—‘οΈ  Clear System")
    print("8. πŸ§ͺ Test OCR")
    print("9. ❓ Help")
    print("0. πŸšͺ Exit")
    print("-" * 40)

def get_user_choice() -> str:
    """Get user choice from menu"""
    try:
        choice = input("\n🎯 Enter your choice (1-9): ").strip()
        return choice
    except KeyboardInterrupt:
        print("\nπŸ‘‹ Goodbye!")
        sys.exit(0)

def simple_ocr_pdf(file_path: str) -> str:
    """Simple OCR function using pytesseract directly"""
    if not OCR_AVAILABLE:
        print("❌ pytesseract not available. Please install it with: pip install pytesseract")
        return ""
    
    try:
        import fitz  # PyMuPDF
        
        # Open PDF
        doc = fitz.open(file_path)
        all_text = ""
        
        for page_num in range(len(doc)):
            page = doc.load_page(page_num)
            
            # Get page as image
            pix = page.get_pixmap()
            img = Image.frombytes("RGB", [pix.width, pix.height], pix.samples)
            
            # Extract text using OCR
            text = pytesseract.image_to_string(img, lang='eng', config='--psm 6')
            all_text += f"\n\n--- Page {page_num + 1} ---\n{text}"
        
        doc.close()
        return all_text
        
    except Exception as e:
        print(f"❌ OCR error: {e}")
        return ""

def upload_document(rag_system: AdvancedRAGSystem):
    """Handle document upload"""
    print("\nπŸ“„ DOCUMENT UPLOAD")
    print("-" * 40)
    
    try:
        # Get file path
        file_path = input("πŸ“ Enter file path (or 'browse' for file dialog): ").strip()
        
        if file_path.lower() == 'browse':
            try:
                import tkinter as tk
                from tkinter import filedialog
                
                root = tk.Tk()
                root.withdraw()
                
                file_path = filedialog.askopenfilename(
                    title="Select Document",
                    filetypes=[
                        ("All supported", "*.pdf;*.txt;*.docx;*.html;*.eml;*.csv;*.json"),
                        ("PDF files", "*.pdf"),
                        ("Text files", "*.txt"),
                        ("Word documents", "*.docx"),
                        ("HTML files", "*.html"),
                        ("Email files", "*.eml"),
                        ("CSV files", "*.csv"),
                        ("JSON files", "*.json"),
                        ("All files", "*.*")
                    ]
                )
                
                root.destroy()
                
                if not file_path:
                    print("❌ No file selected.")
                    return
                    
            except ImportError:
                print("❌ File browser not available. Please enter the file path manually.")
                return
            except Exception as e:
                print(f"❌ Error opening file browser: {e}")
                return
        
        # Check if file exists
        if not os.path.exists(file_path):
            print(f"❌ File not found: {file_path}")
            return
        
        # Check file type
        supported_extensions = {'.pdf', '.txt', '.docx', '.html', '.htm', '.eml', '.msg', '.csv', '.json'}
        file_extension = Path(file_path).suffix.lower()
        
        if file_extension not in supported_extensions:
            print(f"❌ Unsupported file type: {file_extension}")
            print(f"πŸ’‘ Supported types: {', '.join(supported_extensions)}")
            return
        
        # Ask about OCR for PDF files
        use_ocr = False
        if file_extension == '.pdf':
            if OCR_AVAILABLE:
                ocr_choice = input("πŸ” Use OCR for scanned documents? (y/n, default: n): ").strip().lower()
                use_ocr = ocr_choice in ['y', 'yes']
            else:
                print("⚠️ OCR not available. Processing without OCR.")
        
        print(f"\n⏳ Processing document: {os.path.basename(file_path)}")
        print("πŸ’‘ This may take a few moments...")
        
        # Process document
        start_time = time.time()
        try:
            chunks = rag_system.ingest_document(file_path, use_ocr=use_ocr)
            processing_time = time.time() - start_time
        except Exception as e:
            print(f"❌ Document ingestion failed: {e}")
            print("πŸ’‘ Trying alternative processing method...")
            
            # Fallback: Use document processor directly
            try:
                doc_processor = AdvancedDocumentProcessor()
                chunks = doc_processor.process_document(file_path, use_ocr=use_ocr)
                
                # Add chunks to vector database manually
                if chunks:
                    success = rag_system.vector_database.add_documents(chunks)
                    if success:
                        print("βœ… Document processed using fallback method")
                        processing_time = time.time() - start_time
                    else:
                        print("❌ Failed to add documents to vector database")
                        chunks = []
                else:
                    print("❌ No chunks generated from document")
                    chunks = []
                    processing_time = time.time() - start_time
            except Exception as fallback_error:
                print(f"❌ Fallback processing also failed: {fallback_error}")
                chunks = []
                processing_time = time.time() - start_time
        
        if chunks:
            print(f"βœ… Successfully processed {len(chunks)} chunks in {processing_time:.2f} seconds")
            print(f"πŸ“Š Document chunks created:")
            
            for i, chunk in enumerate(chunks[:5]):  # Show first 5 chunks
                print(f"   Chunk {i+1}: {chunk.chunk_id}")
                print(f"   Type: {chunk.section_type}")
                print(f"   Content preview: {chunk.content[:100]}...")
                print()
            
            if len(chunks) > 5:
                print(f"   ... and {len(chunks) - 5} more chunks")
        else:
            print("❌ Failed to process document")
            if file_extension == '.pdf' and not use_ocr:
                print("πŸ’‘ Try using OCR for scanned PDFs: Select 'y' when asked about OCR")
            
    except KeyboardInterrupt:
        print("\nπŸ‘‹ Upload cancelled.")
    except Exception as e:
        print(f"❌ Error uploading document: {e}")

def process_query(rag_system: AdvancedRAGSystem):
    """Interactive query processing with continuous questioning"""
    print("\nπŸ€” INTERACTIVE QUERY PROCESSING")
    print("-" * 40)
    
    print("πŸ’‘ Enter your questions about the uploaded documents.")
    print("πŸ’‘ Type 'quit' or 'exit' to stop asking questions.")
    print("πŸ’‘ Type 'help' for example questions.")
    print()
    
    results = []
    query_count = 0
    
    while True:
        try:
            # Get user input
            user_query = input("πŸ€” Enter your question: ").strip()
            
            # Check for exit commands
            if user_query.lower() in ['quit', 'exit', 'q']:
                print("πŸ‘‹ Goodbye!")
                break
            
            # Check for help command
            if user_query.lower() == 'help':
                print("\nπŸ“‹ Example questions you can ask:")
                print("   β€’ Is heart surgery covered under this policy?")
                print("   β€’ What is the waiting period for pre-existing diseases?")
                print("   β€’ Can I claim for dental treatment?")
                print("   β€’ What is the maximum coverage amount?")
                print("   β€’ Are there any exclusions for chronic diseases?")
                print("   β€’ What documents are required for claim submission?")
                print("   β€’ Is cancer treatment covered?")
                print("   β€’ What is the claim process?")
                print("   β€’ Does the policy cover newborn care after hospital discharge?")
                print()
                continue
            
            # Skip empty queries
            if not user_query:
                print("⚠️  Please enter a question.")
                continue
            
            query_count += 1
            print(f"\nπŸ” Processing Query #{query_count}")
            print(f"πŸ€” Query: {user_query}")
            
            # Process the query
            start_time = time.time()
            try:
                result = rag_system.process_query(user_query)
                processing_time = time.time() - start_time
            except Exception as e:
                print(f"❌ Query processing failed: {e}")
                print("πŸ’‘ Creating fallback result...")
                
                # Create fallback result
                from query_parser import AdvancedQueryParser
                from llm_reasoning import AdvancedLLMReasoning
                
                try:
                    # Parse query
                    query_parser = AdvancedQueryParser()
                    parsed_query = query_parser.parse_query(user_query)
                    
                    # Create fallback reasoning
                    reasoning_engine = AdvancedLLMReasoning()
                    fallback_context = [{
                        'content': f"Based on the query: {user_query}",
                        'source_file': 'fallback',
                        'similarity_score': 0.5
                    }]
                    
                    reasoning_result = reasoning_engine.analyze_query(
                        user_query, fallback_context, parsed_query.query_type
                    )
                    
                    # Create fallback result
                    from rag_system import QueryResult
                    result = QueryResult(
                        query=user_query,
                        parsed_query=parsed_query,
                        search_results=[],
                        reasoning_result=reasoning_result,
                        processing_time=time.time() - start_time,
                        timestamp=time.time(),
                        audit_trail={'status': 'fallback', 'error': str(e)}
                    )
                    processing_time = time.time() - start_time
                    
                except Exception as fallback_error:
                    print(f"❌ Fallback processing also failed: {fallback_error}")
                    continue
        
            # Display results
            print(f"\nπŸ“‹ QUERY RESULTS")
            print("=" * 50)
            print(f"⏱️  Processing time: {processing_time:.2f} seconds")
            print(f"🎯 Decision: {result.reasoning_result.decision.upper()}")
            print(f"πŸ“Š Confidence: {result.reasoning_result.confidence_score:.1%}")
            
            if result.reasoning_result.amount:
                print(f"πŸ’° Amount: ${result.reasoning_result.amount:,.2f}")
            
            if result.reasoning_result.waiting_period:
                print(f"⏰ Waiting Period: {result.reasoning_result.waiting_period}")
            
            print(f"\nπŸ“ Justification:")
            print(result.reasoning_result.justification)
            
            if result.reasoning_result.relevant_clauses:
                print(f"\nπŸ“„ Relevant Clauses:")
                for clause in result.reasoning_result.relevant_clauses:
                    print(f"   β€’ {clause}")
            
            if result.reasoning_result.conditions:
                print(f"\nβœ… Conditions:")
                for condition in result.reasoning_result.conditions:
                    print(f"   β€’ {condition}")
            
            if result.reasoning_result.exclusions:
                print(f"\n❌ Exclusions:")
                for exclusion in result.reasoning_result.exclusions:
                    print(f"   β€’ {exclusion}")
            
            if result.reasoning_result.required_documents:
                print(f"\nπŸ“‹ Required Documents:")
                for doc in result.reasoning_result.required_documents:
                    print(f"   β€’ {doc}")
            
            # Show search results summary
            if result.search_results:
                print(f"\nπŸ” Search Results Summary:")
                print(f"   Found {len(result.search_results)} relevant documents")
                for i, search_result in enumerate(result.search_results[:3], 1):
                    print(f"   {i}. {search_result.source_file} (Score: {search_result.similarity_score:.2f})")
            
            # Ask if user wants to see detailed explanation
            show_details = input("\n❓ Show detailed explanation? (y/n): ").strip().lower()
            if show_details in ['y', 'yes']:
                try:
                    detailed_explanation = rag_system.reasoning_engine.explain_decision(result.reasoning_result)
                    print(f"\nπŸ“– DETAILED EXPLANATION:")
                    print("=" * 50)
                    print(detailed_explanation)
                except Exception as e:
                    print(f"❌ Error generating detailed explanation: {e}")
                    print("πŸ’‘ Detailed explanation not available")
            
            results.append({
                "query": user_query,
                "result": result,
                "processing_time": processing_time
            })
            
            print()
            
            # Ask if user wants to continue
            if query_count % 3 == 0:  # Ask every 3 queries
                continue_choice = input("❓ Continue asking questions? (y/n): ").strip().lower()
                if continue_choice not in ['y', 'yes', '']:
                    print("πŸ‘‹ Thanks for using the RAG system!")
                    break
        
        except KeyboardInterrupt:
            print("\nπŸ‘‹ Interrupted by user. Goodbye!")
            break
        except Exception as e:
            print(f"❌ Error processing query: {e}")
            print("πŸ’‘ Try asking a different question or type 'help' for examples.")
            print()

    return results

def show_system_statistics(rag_system: AdvancedRAGSystem):
    """Show system statistics"""
    print("\nπŸ“Š SYSTEM STATISTICS")
    print("-" * 40)
    
    try:
        stats = rag_system.get_system_statistics()
        
        if stats:
            # Vector database stats
            db_stats = stats.get('vector_database', {})
            print(f"πŸ“š Vector Database:")
            print(f"   Total chunks: {db_stats.get('total_chunks', 0)}")
            print(f"   Unique sources: {db_stats.get('unique_sources', 0)}")
            print(f"   File types: {', '.join(db_stats.get('file_types', []))}")
            
            # Audit trail stats
            audit_stats = stats.get('audit_trail', {})
            print(f"\nπŸ“‹ Audit Trail:")
            print(f"   Total entries: {audit_stats.get('total_entries', 0)}")
            print(f"   Successful queries: {audit_stats.get('successful_queries', 0)}")
            print(f"   Failed queries: {audit_stats.get('failed_queries', 0)}")
            print(f"   Document ingestions: {audit_stats.get('document_ingestions', 0)}")
            
            # Component info
            components = stats.get('components', {})
            print(f"\nπŸ”§ Components:")
            print(f"   Document Processor: {components.get('document_processor', 'Unknown')}")
            print(f"   Vector Database: {components.get('vector_database', 'Unknown')}")
            print(f"   Query Parser: {components.get('query_parser', 'Unknown')}")
            print(f"   Reasoning Engine: {components.get('reasoning_engine', 'Unknown')}")
            print(f"   GPU Enabled: {components.get('use_gpu', False)}")
        else:
            print("❌ Unable to retrieve system statistics")
            
    except Exception as e:
        print(f"❌ Error getting system statistics: {e}")

def show_audit_trail(rag_system: AdvancedRAGSystem):
    """Show audit trail"""
    print("\nπŸ” AUDIT TRAIL")
    print("-" * 40)
    
    try:
        audit_log = rag_system.get_audit_trail()
        
        if audit_log:
            print(f"πŸ“‹ Total entries: {len(audit_log)}")
            
            # Show recent entries
            recent_entries = audit_log[-10:]  # Last 10 entries
            print(f"\nπŸ“ Recent Entries:")
            
            for i, entry in enumerate(reversed(recent_entries), 1):
                action = entry.get('action', 'Unknown')
                timestamp = entry.get('timestamp', 'Unknown')
                status = entry.get('status', 'Unknown')
                
                print(f"   {i}. {action} - {status} ({timestamp})")
                
                if action == 'query_processing':
                    query = entry.get('query', 'Unknown')
                    print(f"      Query: {query[:50]}...")
                    
                    reasoning = entry.get('reasoning_result', {})
                    if reasoning:
                        decision = reasoning.get('decision', 'Unknown')
                        confidence = reasoning.get('confidence_score', 0.0)
                        print(f"      Decision: {decision} (Confidence: {confidence:.1%})")
                
                elif action == 'document_ingestion':
                    file_path = entry.get('file_path', 'Unknown')
                    chunks = entry.get('chunks_processed', 0)
                    print(f"      File: {os.path.basename(file_path)} ({chunks} chunks)")
                
                print()
            
            # Ask if user wants to save audit trail
            save_choice = input("πŸ’Ύ Save audit trail to file? (y/n): ").strip().lower()
            if save_choice in ['y', 'yes']:
                filename = input("πŸ“ Enter filename (default: audit_trail.json): ").strip()
                if not filename:
                    filename = "audit_trail.json"
                
                if rag_system.save_audit_trail(filename):
                    print(f"βœ… Audit trail saved to: {filename}")
                else:
                    print("❌ Failed to save audit trail")
        else:
            print("πŸ“‹ No audit trail entries found")
            
    except Exception as e:
        print(f"❌ Error showing audit trail: {e}")

def validate_system(rag_system: AdvancedRAGSystem):
    """Validate system components"""
    print("\nπŸ§ͺ SYSTEM VALIDATION")
    print("-" * 40)
    
    try:
        validation = rag_system.validate_system()
        
        print("πŸ” Checking system components...")
        
        components = [
            ('Document Processor', validation.get('document_processor', False)),
            ('Vector Database', validation.get('vector_database', False)),
            ('Query Parser', validation.get('query_parser', False)),
            ('Reasoning Engine', validation.get('reasoning_engine', False))
        ]
        
        all_valid = True
        for component_name, is_valid in components:
            status = "βœ… PASS" if is_valid else "❌ FAIL"
            print(f"   {component_name}: {status}")
            if not is_valid:
                all_valid = False
        
        print(f"\n🎯 Overall Status: {'βœ… ALL COMPONENTS VALID' if all_valid else '❌ SOME COMPONENTS FAILED'}")
        
        if not all_valid:
            print("\n❌ Errors found:")
            for error in validation.get('errors', []):
                print(f"   β€’ {error}")
        
    except Exception as e:
        print(f"❌ Error validating system: {e}")

def export_system_data(rag_system: AdvancedRAGSystem):
    """Export system data"""
    print("\nπŸ’Ύ EXPORT SYSTEM DATA")
    print("-" * 40)
    
    try:
        filename = input("πŸ“ Enter filename (default: system_export.json): ").strip()
        if not filename:
            filename = "system_export.json"
        
        print(f"⏳ Exporting system data to: {filename}")
        
        if rag_system.export_system_data(filename):
            print(f"βœ… System data exported successfully to: {filename}")
            
            # Show file size
            if os.path.exists(filename):
                file_size = os.path.getsize(filename)
                print(f"πŸ“ File size: {file_size:,} bytes")
        else:
            print("❌ Failed to export system data")
            
    except Exception as e:
        print(f"❌ Error exporting system data: {e}")

def clear_system(rag_system: AdvancedRAGSystem):
    """Clear system data"""
    print("\nπŸ—‘οΈ  CLEAR SYSTEM")
    print("-" * 40)
    
    try:
        confirm = input("⚠️  This will clear ALL system data. Are you sure? (yes/no): ").strip().lower()
        
        if confirm == 'yes':
            print("⏳ Clearing system data...")
            
            if rag_system.clear_system():
                print("βœ… System data cleared successfully")
            else:
                print("❌ Failed to clear system data")
        else:
            print("❌ Operation cancelled")
            
    except Exception as e:
        print(f"❌ Error clearing system: {e}")

def test_ocr():
    """Test OCR functionality"""
    print("\nπŸ§ͺ OCR TEST")
    print("-" * 40)
    
    if not OCR_AVAILABLE:
        print("❌ pytesseract not available")
        print("πŸ’‘ Install with: pip install pytesseract")
        return
    
    try:
        # Create a simple test image
        from PIL import Image, ImageDraw, ImageFont
        
        # Create test image
        img = Image.new('RGB', (300, 100), color='white')
        draw = ImageDraw.Draw(img)
        
        # Try to use a default font
        try:
            font = ImageFont.load_default()
        except:
            font = None
        
        # Draw text
        text = "Test OCR Text"
        draw.text((10, 40), text, fill='black', font=font)
        
        # Test OCR
        ocr_text = pytesseract.image_to_string(img)
        print(f"βœ… OCR test successful")
        print(f"   Original: '{text}'")
        print(f"   OCR result: '{ocr_text.strip()}'")
        
    except Exception as e:
        print(f"❌ OCR test failed: {e}")
        print("πŸ’‘ Make sure Tesseract is installed and configured")

def show_help():
    """Show help information"""
    print("\n❓ HELP")
    print("-" * 40)
    print("πŸ“‹ This RAG system can process various document types and answer questions about them.")
    print("\nπŸ“„ Supported Document Types:")
    print("   β€’ PDF files (with OCR support for scanned documents)")
    print("   β€’ Text files (.txt)")
    print("   β€’ Word documents (.docx)")
    print("   β€’ HTML files (.html)")
    print("   β€’ Email files (.eml, .msg)")
    print("   β€’ CSV files (.csv)")
    print("   β€’ JSON files (.json)")
    
    print("\nπŸ€” Query Examples:")
    print("   β€’ 'Is heart surgery covered under this policy?'")
    print("   β€’ 'How do I file a claim?'")
    print("   β€’ 'What is the waiting period for pre-existing conditions?'")
    print("   β€’ 'What documents are required for claim submission?'")
    print("   β€’ 'What is the maximum coverage amount?'")
    
    print("\nπŸ’‘ Tips:")
    print("   β€’ Upload documents first before asking questions")
    print("   β€’ Use natural language - the system understands plain English")
    print("   β€’ The system can handle vague or incomplete queries")
    print("   β€’ All queries are logged in the audit trail")
    print("   β€’ Use the system validation to check component status")
    print("   β€’ Test OCR functionality if you have issues with scanned documents")

def main():
    """Main function"""
    # Parse command line arguments
    parser = argparse.ArgumentParser(description='Advanced RAG System')
    parser.add_argument('--query', help='Process a single query from file')
    parser.add_argument('--upload', help='Upload and process a document')
    parser.add_argument('--status', action='store_true', help='Check system status')
    
    args = parser.parse_args()
    
    # Handle command line arguments
    if args.query:
        # Process single query from file
        try:
            with open(args.query, 'r') as f:
                question = f.read().strip()
            
            # Initialize RAG system
            rag_system = AdvancedRAGSystem(use_gpu=False)
            
            # Process the query
            result = rag_system.process_query(question)
            
            # Output results in a structured format
            print(f"Question: {question}")
            print(f"Decision: {result.reasoning_result.decision}")
            print(f"Confidence: {result.reasoning_result.confidence_score:.1%}")
            print(f"Justification: {result.reasoning_result.justification}")
            
            if result.reasoning_result.amount:
                print(f"Amount: ${result.reasoning_result.amount:,.2f}")
            
            if result.reasoning_result.waiting_period:
                print(f"Waiting Period: {result.reasoning_result.waiting_period}")
            
            if result.reasoning_result.relevant_clauses:
                print(f"Relevant Clauses: {', '.join(result.reasoning_result.relevant_clauses)}")
            
            if result.reasoning_result.conditions:
                print(f"Conditions: {', '.join(result.reasoning_result.conditions)}")
            
            if result.reasoning_result.exclusions:
                print(f"Exclusions: {', '.join(result.reasoning_result.exclusions)}")
            
            if result.reasoning_result.required_documents:
                print(f"Required Documents: {', '.join(result.reasoning_result.required_documents)}")
            
            return
        except Exception as e:
            print(f"Error processing query: {e}")
            sys.exit(1)
    
    elif args.upload:
        # Upload and process document
        try:
            # Initialize RAG system
            rag_system = AdvancedRAGSystem(use_gpu=False)
            
            # Process the document
            chunks = rag_system.ingest_document(args.upload, use_ocr=False)
            
            print(f"Document processed successfully: {args.upload}")
            print(f"Chunks processed: {len(chunks)}")
            
            return
        except Exception as e:
            print(f"Error processing document: {e}")
            sys.exit(1)
    
    elif args.status:
        # Check system status
        try:
            rag_system = AdvancedRAGSystem(use_gpu=False)
            print("System Status: READY")
            return
        except Exception as e:
            print(f"System Status: ERROR - {e}")
            sys.exit(1)
    
    # Interactive mode (default)
    print_banner()
    
    try:
        # Initialize RAG system
        print("πŸš€ Initializing RAG system...")
        try:
            rag_system = AdvancedRAGSystem(use_gpu=False)  # Use CPU for better compatibility
            print("βœ… RAG system initialized successfully!")
        except Exception as e:
            print(f"❌ Failed to initialize RAG system: {e}")
            print("πŸ’‘ Trying with minimal configuration...")
            
            try:
                # Try with minimal settings
                rag_system = AdvancedRAGSystem(
                    use_gpu=False,
                    model_path=None  # Don't require model file
                )
                print("βœ… RAG system initialized with minimal configuration!")
            except Exception as e2:
                print(f"❌ RAG system initialization failed: {e2}")
                print("πŸ’‘ Please check your system configuration")
                return
        
        # Main loop
        while True:
            print_menu()
            choice = get_user_choice()
            
            if choice == '1':
                upload_document(rag_system)
            elif choice == '2':
                process_query(rag_system)
            elif choice == '3':
                show_system_statistics(rag_system)
            elif choice == '4':
                show_audit_trail(rag_system)
            elif choice == '5':
                validate_system(rag_system)
            elif choice == '6':
                export_system_data(rag_system)
            elif choice == '7':
                clear_system(rag_system)
            elif choice == '8':
                test_ocr()
            elif choice == '9':
                show_help()
            elif choice == '0':
                print("πŸ‘‹ Thank you for using the Advanced RAG System!")
                break
            else:
                print("❌ Invalid choice. Please enter a number between 1-0.")
            
            # Pause before showing menu again
            input("\n⏸️  Press Enter to continue...")
    
    except KeyboardInterrupt:
        print("\nπŸ‘‹ Goodbye!")
    except Exception as e:
        print(f"❌ Fatal error: {e}")
        print("πŸ’‘ Please check your system configuration and try again.")

if __name__ == "__main__":
    main()