File size: 4,882 Bytes
09281fe | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 | #!/usr/bin/env python3
"""
Test script to isolate document processing issues
"""
import os
import sys
import time
from pathlib import Path
def test_document_processor():
"""Test the document processor directly"""
print("🔍 TESTING DOCUMENT PROCESSOR")
print("=" * 40)
try:
from document_processer import AdvancedDocumentProcessor
# Initialize processor
processor = AdvancedDocumentProcessor()
print("✅ Document processor initialized")
# Test with doc2.pdf
file_path = "doc2.pdf"
if not os.path.exists(file_path):
print(f"❌ File not found: {file_path}")
return
print(f"📄 Processing file: {file_path}")
# Test without OCR
print("\n--- Testing without OCR ---")
start_time = time.time()
try:
chunks = processor.process_document(file_path, use_ocr=False)
processing_time = time.time() - start_time
print(f"✅ Success! Processed {len(chunks)} chunks in {processing_time:.2f}s")
# Show first chunk
if chunks:
print(f"📋 First chunk preview:")
print(f" ID: {chunks[0].chunk_id}")
print(f" Content: {chunks[0].content[:100]}...")
print(f" Source: {chunks[0].source_file}")
except Exception as e:
print(f"❌ Failed without OCR: {e}")
# Test with OCR
print("\n--- Testing with OCR ---")
start_time = time.time()
try:
chunks = processor.process_document(file_path, use_ocr=True)
processing_time = time.time() - start_time
print(f"✅ Success! Processed {len(chunks)} chunks in {processing_time:.2f}s")
# Show first chunk
if chunks:
print(f"📋 First chunk preview:")
print(f" ID: {chunks[0].chunk_id}")
print(f" Content: {chunks[0].content[:100]}...")
print(f" Source: {chunks[0].source_file}")
except Exception as e:
print(f"❌ Failed with OCR: {e}")
except Exception as e:
print(f"❌ Error initializing document processor: {e}")
def test_vector_database():
"""Test the vector database directly"""
print("\n🔍 TESTING VECTOR DATABASE")
print("=" * 40)
try:
from vector_database import VectorDatabase
# Initialize vector database
vector_db = VectorDatabase()
print("✅ Vector database initialized")
# Test adding a simple document
test_content = "This is a test document for vector database testing."
test_metadata = {
'source_file': 'test.txt',
'file_type': 'text',
'section_type': 'test'
}
print("📝 Adding test document...")
success = vector_db.add_document(test_content, test_metadata)
if success:
print("✅ Successfully added test document")
# Test search
print("🔍 Testing search...")
results = vector_db.search_documents("test document", n_results=3)
print(f"✅ Search returned {len(results)} results")
else:
print("❌ Failed to add test document")
except Exception as e:
print(f"❌ Error with vector database: {e}")
def test_rag_system():
"""Test the RAG system directly"""
print("\n🔍 TESTING RAG SYSTEM")
print("=" * 40)
try:
from rag_system import AdvancedRAGSystem
# Initialize RAG system
print("🔄 Initializing RAG system...")
rag_system = AdvancedRAGSystem(use_gpu=False) # Use CPU for testing
print("✅ RAG system initialized")
# Test document ingestion
file_path = "doc2.pdf"
if os.path.exists(file_path):
print(f"📄 Testing document ingestion: {file_path}")
try:
chunks = rag_system.ingest_document(file_path, use_ocr=False)
print(f"✅ Successfully ingested {len(chunks)} chunks")
except Exception as e:
print(f"❌ Document ingestion failed: {e}")
else:
print(f"❌ File not found: {file_path}")
except Exception as e:
print(f"❌ Error with RAG system: {e}")
def main():
"""Run all tests"""
print("🧪 DOCUMENT PROCESSING DIAGNOSTICS")
print("=" * 50)
# Test 1: Document processor
test_document_processor()
# Test 2: Vector database
test_vector_database()
# Test 3: RAG system
test_rag_system()
print("\n✅ Diagnostics completed!")
if __name__ == "__main__":
main() |