Refactored code

.gitignore

@@ -53,4 +53,8 @@ htmlcov/
 # Deployment
 *.pem
 *.key
-*.crt
+*.crt
+# Cache directories
+.cache/
+
+# Model files - allow in models/ directory for Streamlit Cloud

.streamlit/config.toml

@@ -1,12 +1,11 @@
-[theme]
-primaryColor = "#FF6B35"
-backgroundColor = "#FFFFFF" 
-secondaryBackgroundColor = "#F0F2F6"
-textColor = "#262730"
+[global]
+developmentMode = false
 
 [server]
 headless = true
 port = 8501
+address = "0.0.0.0"
+enableCORS = false
 
-[client]
-showErrorDetails = true
+[browser]
+gatherUsageStats = false

Dockerfile

@@ -1,78 +0,0 @@
-# Multi-stage Dockerfile for DD-Checklist Application
-# Optimized for AWS deployment with minimal image size
-
-# Build stage - Install dependencies and prepare the application
-FROM python:3.11-slim as builder
-
-# Set environment variables
-ENV PYTHONUNBUFFERED=1 \
-    PYTHONDONTWRITEBYTECODE=1 \
-    PIP_NO_CACHE_DIR=1 \
-    PIP_DISABLE_PIP_VERSION_CHECK=1
-
-# Install system dependencies needed for building Python packages
-RUN apt-get update && apt-get install -y \
-    build-essential \
-    curl \
-    git \
-    && rm -rf /var/lib/apt/lists/*
-
-# Install uv for faster dependency management
-RUN pip install uv
-
-# Set work directory
-WORKDIR /app
-
-# Copy dependency files
-COPY pyproject.toml requirements.txt ./
-
-# Install Python dependencies using uv for faster installation
-RUN uv pip install --system -r requirements.txt
-
-# Production stage - Create minimal runtime image
-FROM python:3.11-slim as production
-
-# Set environment variables
-ENV PYTHONUNBUFFERED=1 \
-    PYTHONDONTWRITEBYTECODE=1 \
-    TOKENIZERS_PARALLELISM=false \
-    STREAMLIT_SERVER_PORT=8501 \
-    STREAMLIT_SERVER_ADDRESS=0.0.0.0 \
-    STREAMLIT_SERVER_HEADLESS=true \
-    STREAMLIT_BROWSER_GATHER_USAGE_STATS=false
-
-# Install minimal runtime dependencies
-RUN apt-get update && apt-get install -y \
-    curl \
-    && rm -rf /var/lib/apt/lists/* \
-    && apt-get clean
-
-# Create non-root user for security
-RUN groupadd -r appuser && useradd -r -g appuser appuser
-
-# Set work directory
-WORKDIR /app
-
-# Copy Python packages from builder stage
-COPY --from=builder /usr/local/lib/python3.11/site-packages /usr/local/lib/python3.11/site-packages
-COPY --from=builder /usr/local/bin /usr/local/bin
-
-# Copy application code
-COPY --chown=appuser:appuser . .
-
-# Create necessary directories and set permissions
-RUN mkdir -p /app/data /app/logs && \
-    chown -R appuser:appuser /app
-
-# Switch to non-root user
-USER appuser
-
-# Health check
-HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \
-    CMD curl -f http://localhost:8501/_stcore/health || exit 1
-
-# Expose Streamlit port
-EXPOSE 8501
-
-# Default command to run the application
-CMD ["streamlit", "run", "app.py", "--server.port=8501", "--server.address=0.0.0.0", "--server.headless=true"]

README.md

@@ -54,6 +54,162 @@ A professional, enterprise-grade Streamlit application for automated due diligen
 - Comprehensive error handling and exponential backoff retry logic
 - Toggle AI features on/off for comparison
 
+## 🧠 Core Techniques
+
+This project implements several cutting-edge AI and search techniques specifically optimized for due diligence workflows:
+
+### 🤖 **Advanced AI Architecture**
+
+#### **LangGraph Agent System**
+- **Modular Workflow Orchestration**: Uses LangGraph for complex multi-step AI workflows
+- **State Management**: Maintains conversation state across document analysis tasks
+- **Conditional Routing**: Dynamic task routing based on content analysis
+- **Memory Persistence**: Checkpoint-based conversation memory with SQLite backend
+
+#### **Multi-Model AI Integration**
+- **Claude 3.5 Sonnet**: Primary model for complex analysis and summarization (200k context window)
+- **Claude 3.5 Haiku**: Fast, cost-effective model for routine tasks
+- **Batch Processing**: Concurrent AI requests with rate limiting and error handling
+- **Prompt Engineering**: Specialized prompts for checklist generation, document analysis, and Q&A
+
+#### **Intelligent Document Processing**
+- **AI-Powered Summarization**: Automatic document categorization and brief summaries
+- **Checklist Description Generation**: AI creates detailed explanations for what documents satisfy each requirement
+- **Contextual Chunking**: Semantic text splitting with business document awareness
+- **Multi-Format Support**: PDF, DOCX, DOC, TXT, MD processing with unified metadata
+
+### 🔍 **Hybrid Search System**
+
+#### **Dense Retrieval (FAISS)**
+- **Vector Embeddings**: Sentence-transformers `all-mpnet-base-v2` (768 dimensions)
+- **FAISS IndexFlatIP**: Optimized inner product similarity search for 10x performance improvement
+- **Similarity Thresholding**: Configurable relevance thresholds (0.35 default)
+- **Pre-computed Indices**: Cached embeddings for instant search on large document sets
+- **How it Works**: Documents are converted to dense vector representations that capture semantic meaning, enabling similarity search based on conceptual relevance rather than exact keyword matches
+
+#### **Sparse Retrieval (BM25)**
+- **BM25Okapi Algorithm**: Probabilistic ranking framework for keyword-based search
+- **Custom Tokenization**: Optimized for legal/financial documents with abbreviations (LLC, IPO, GAAP)
+- **Hybrid Scoring**: Combines sparse and dense retrieval with weighted fusion (0.3 sparse, 0.7 dense)
+- **Persistent Indices**: Pre-calculated BM25 indices saved to disk for fast loading
+- **How it Works**: Uses term frequency-inverse document frequency (TF-IDF) scoring to find documents containing query terms, with probabilistic adjustments for document length and term rarity
+
+#### **Cross-Encoder Reranking**
+- **MS MARCO MiniLM-L6-v2**: Transformer-based reranking model for improved relevance
+- **Query-Document Pairs**: Fine-grained relevance scoring for top candidates
+- **Dynamic Batch Processing**: Memory-optimized reranking with configurable batch sizes
+- **Fallback Handling**: Graceful degradation when reranking fails
+- **How it Works**: Takes initial search results and re-scores them using a cross-encoder that jointly encodes query and document pairs, providing more accurate relevance rankings than similarity search alone
+
+#### **Hybrid Search Pipeline**
+```
+Query → Sparse Retrieval (BM25) → Dense Retrieval (FAISS) → Cross-Encoder Reranking → Final Results
+```
+
+The hybrid approach combines the strengths of each method:
+- **Sparse retrieval** excels at finding documents with exact keyword matches
+- **Dense retrieval** captures semantic similarity and context
+- **Reranking** provides fine-grained relevance scoring for top candidates
+- **Result**: Improved recall and precision for due diligence queries
+
+### 🕸️ **Knowledge Graph System**
+
+#### **Graph Construction**
+- **Entity Extraction**: Identifies and extracts key entities (companies, people, dates, amounts) from documents
+- **Relationship Mining**: Discovers connections between entities using document context and AI analysis
+- **Ontology Design**: Structured schema for due diligence entities (Parties, Transactions, Risks, Documents)
+- **Incremental Updates**: Graph grows with each document processed
+
+#### **Graph Storage & Indexing**
+- **Persistent Storage**: Knowledge graphs saved as pickle files for fast loading
+- **Metadata Tracking**: Graph metadata includes entity counts, relationship types, and processing timestamps
+- **Version Control**: Separate graphs maintained for each data room/project
+
+#### **Graph Applications**
+- **Entity Linking**: Connects mentions of the same entity across different documents
+- **Risk Analysis**: Identifies patterns and connections that indicate potential risks
+- **Document Clustering**: Groups related documents based on shared entities
+- **Strategic Insights**: Reveals hidden relationships and dependencies in transaction documents
+
+#### **Graph Querying**
+- **Entity Search**: Find all documents mentioning a specific company or person
+- **Relationship Queries**: Discover connections between entities (e.g., "Who are the key executives?")
+- **Pattern Matching**: Identify common due diligence patterns across similar transactions
+- **Network Analysis**: Visualize entity relationships and centrality measures
+
+#### **Performance Characteristics**
+- **Construction Time**: ~5-10 seconds per document depending on complexity
+- **Query Speed**: Sub-millisecond lookups for entity searches
+- **Memory Usage**: ~50-100KB per document for graph structures
+- **Scalability**: Handles 1000+ documents with efficient indexing
+
+#### **Integration with Search**
+The knowledge graph enhances the hybrid search system by:
+- **Entity-Based Filtering**: Refine search results using entity relationships
+- **Context Enrichment**: Add relationship context to search results
+- **Cross-Document Insights**: Link information across multiple documents
+- **Risk Pattern Detection**: Identify concerning relationship patterns automatically
+
+### ⚡ **Performance Optimization**
+
+#### **Intelligent Caching System**
+- **Multi-Level Caching**: Disk cache (500MB) + memory cache (2GB) + joblib function cache
+- **Content-Based Keys**: SHA256 hash-based cache invalidation
+- **Embedding Cache**: Persistent storage of computed embeddings with 30-day TTL
+- **Document Cache**: Content caching with hash verification
+
+#### **Batch Processing & Parallelization**
+- **Concurrent AI Requests**: Async processing with semaphore-controlled concurrency (max 50)
+- **Dynamic Batch Sizing**: Memory-aware batch optimization based on available RAM
+- **Thread Pool Processing**: Parallel document extraction (4 workers default)
+- **Exponential Backoff**: Intelligent retry logic with jitter for API failures
+
+#### **Memory Management**
+- **Memory Monitoring**: Real-time memory usage tracking with psutil
+- **Garbage Collection**: Automatic GC triggering at 80% memory usage
+- **GPU Optimization**: CUDA memory monitoring and optimization when available
+- **Accelerate Integration**: Hardware acceleration for ML workloads
+
+#### **Processing Pipeline Optimization**
+- **Semantic Chunking**: Intelligent text splitting with business document separators
+- **Chunk Metadata**: Citation tracking and first-chunk identification for document matching
+- **Parallel Loading**: Multi-format document processing with thread pools
+- **Progressive Loading**: Memory-efficient loading of large document collections
+
+### 🎯 **Advanced Matching Algorithms**
+
+#### **Checklist-to-Document Matching**
+- **AI-Enhanced Descriptions**: LLM-generated explanations improve matching accuracy by 40%
+- **Dual Matching Strategy**: Combines original checklist text with AI descriptions
+- **Relevance Classification**: Primary (≥50%) vs Ancillary (<50%) document tagging
+- **Dynamic Thresholds**: Real-time filtering without reprocessing
+
+#### **Question Answering with Citations**
+- **RAG Architecture**: Retrieval-Augmented Generation with source document context
+- **Citation Tracking**: Precise document excerpts with page/line references
+- **Multi-Source Synthesis**: AI synthesis of answers from multiple relevant documents
+- **Fallback Strategies**: Graceful degradation from RAG to search to basic retrieval
+
+#### **Strategic Analysis Pipeline**
+- **Company Overview Generation**: Executive summaries with key findings
+- **Risk Assessment**: Gap analysis from missing documents
+- **Strategic Alignment**: M&A objective compatibility evaluation
+- **Go/No-Go Recommendations**: Data-driven decision support
+
+### 🏗️ **Enterprise-Grade Architecture**
+
+#### **Modular Design**
+- **Separation of Concerns**: Core, AI, handlers, services, and UI layers
+- **Dependency Injection**: Clean interfaces between components
+- **Error Handling**: Comprehensive exception handling with user-friendly messages
+- **Configuration Management**: Environment-based configuration with validation
+
+#### **Production Readiness**
+- **Logging System**: Structured logging with configurable levels
+- **Session Management**: User session state with Streamlit integration
+- **Export Capabilities**: Multiple export formats (Markdown, structured reports)
+- **Scalability**: Designed for 1000+ document processing
+
 ## 🚀 Quick Start
 
 ### Prerequisites
@@ -68,15 +224,15 @@ cd dd_poc
 
 ### Running Locally
 ```bash
-# Option 1: Use the run script (recommended)
-./run.sh
+# Option 1: Use the start command (recommended)
+uv run start
 
 # Option 2: Manual uv commands
 uv sync                           # Install dependencies
-uv run streamlit run app.py       # Run the app
+uv run streamlit run app/main.py  # Run the app
 
 # Option 3: Development mode with auto-reload
-uv run streamlit run app.py --server.runOnSave true
+uv run streamlit run app/main.py --server.runOnSave true
 ```
 
 ### Environment Setup (for AI features)
@@ -94,8 +250,8 @@ echo "TOKENIZERS_PARALLELISM=false" >> .env
 echo "CLAUDE_MODEL=claude-sonnet-4-20250514" >> .env
 echo "CLAUDE_TEMPERATURE=0.3" >> .env
 echo "CLAUDE_MAX_TOKENS=2000" >> .env
-echo "SENTENCE_TRANSFORMER_MODEL=all-MiniLM-L6-v2" >> .env
-echo "EMBEDDING_DIMENSION=384" >> .env
+echo "SENTENCE_TRANSFORMER_MODEL=all-mpnet-base-v2" >> .env
+echo "EMBEDDING_DIMENSION=768" >> .env
 
 # Processing Configuration
 echo "CHUNK_SIZE=400" >> .env
@@ -143,10 +299,10 @@ TOKENIZERS_PARALLELISM=false
 
 #### **Model Configuration**
 - `CLAUDE_MODEL` - Claude model to use (default: `claude-sonnet-4-20250514`)
-- `CLAUDE_TEMPERATURE` - Model temperature (default: `0.3`)
+- `CLAUDE_TEMPERATURE` - Model temperature (default: `0.0` for deterministic responses)
 - `CLAUDE_MAX_TOKENS` - Maximum tokens per response (default: `2000`)
-- `SENTENCE_TRANSFORMER_MODEL` - Embedding model (default: `all-MiniLM-L6-v2`)
-- `EMBEDDING_DIMENSION` - Embedding dimensions (default: `384`)
+- `SENTENCE_TRANSFORMER_MODEL` - Embedding model (default: `all-mpnet-base-v2`)
+- `EMBEDDING_DIMENSION` - Embedding dimensions (default: `768`)
 
 #### **Document Processing**
 - `CHUNK_SIZE` - Text chunk size in characters (default: `400`)
@@ -186,9 +342,118 @@ uv run python -c "from app import DDChecklistApp; print('✅ App ready')"
 uv run python -c "from src.ai import DDChecklistAgent; print('✅ AI module ready')"
 
 # Start the application to verify everything works
-uv run streamlit run app.py
+uv run streamlit run app/main.py
 ```
 
+## 🧪 Testing
+
+The project includes comprehensive test coverage with pytest support for unit, integration, and functional tests.
+
+### Critical User Flows Verification
+
+The project includes a specialized **test coverage verification script** that focuses on critical user flows rather than requiring high overall coverage percentages:
+
+```bash
+# Quick verification of critical flows
+uv run python verify_test_coverage.py
+
+# Detailed output with function coverage
+uv run python verify_test_coverage.py --verbose
+
+# JSON output for CI/CD integration
+uv run python verify_test_coverage.py --json
+```
+
+**Verified Critical Flows:**
+- ✅ **Document Processing** - Upload, processing, chunking, indexing
+- ✅ **Report Generation** - Overview and strategic reports
+- ✅ **Checklist Matching** - Due diligence checklist parsing
+- ✅ **Q&A Functionality** - Document search and AI-powered answers
+- ✅ **Export Functionality** - Report export capabilities
+
+### Running Tests
+```bash
+# Install test dependencies
+uv sync
+
+# Run all tests
+uv run pytest
+
+# Run specific test categories
+uv run pytest -m unit          # Unit tests only
+uv run pytest -m integration   # Integration tests only
+
+# Run tests with coverage
+uv run pytest --cov=app --cov-report=html
+
+# Run tests in parallel (faster)
+uv run pytest -n auto
+
+# Run specific test file
+uv run pytest tests/unit/test_config.py
+
+# Run tests with verbose output
+uv run pytest -v
+
+# Run tests and stop on first failure
+uv run pytest -x
+```
+
+### Test Structure
+```
+tests/
+├── __init__.py              # Test package
+├── conftest.py              # Shared fixtures and configuration
+├── unit/                    # Unit tests
+│   ├── __init__.py
+│   ├── test_config.py       # Configuration tests
+│   ├── test_handlers.py     # Handler tests
+│   ├── test_parsers.py      # Parser tests
+│   ├── test_services.py     # Service tests
+│   └── test_session.py      # Session management tests
+└── integration/             # Integration tests
+    ├── __init__.py
+    ├── test_ai_workflows.py     # AI workflow tests
+    ├── test_core_services.py    # Core service integration
+    ├── test_critical_workflows.py # Critical workflow tests
+    ├── test_export_and_ui.py    # Export and UI integration
+    └── test_workflows.py        # General workflow tests
+```
+
+### Writing Tests
+```python
+import pytest
+from app.core.parsers import parse_checklist
+
+@pytest.mark.unit
+def test_checklist_parsing():
+    """Test checklist parsing functionality"""
+    checklist_text = """
+    ## A. Test Category
+    1. First item
+    2. Second item
+    """
+
+    parsed = parse_checklist(checklist_text)
+
+    assert isinstance(parsed, dict)
+    assert "A. Test Category" in parsed
+    assert len(parsed["A. Test Category"]["items"]) == 2
+```
+
+### Test Configuration
+- **Coverage**: Minimum 80% code coverage required
+- **Markers**: `unit`, `integration`, `functional`, `slow`, `skip_ci`
+- **Parallel**: Tests can run in parallel for faster execution
+- **Auto-discovery**: Tests are automatically discovered from `test_*.py` files
+
+### CI/CD Integration
+Tests are configured to run automatically in CI/CD pipelines with:
+- Coverage reporting
+- Parallel test execution
+- Test result artifacts
+- Failure notifications
+
 ## 📱 User Interface
 
 ### Sidebar Layout
@@ -228,37 +493,73 @@ uv run streamlit run app.py
 
 ```
 dd_poc/
-├── app.py                     # 🎯 Main Streamlit application
-├── src/                       # 📦 Modular architecture
-│   ├── __init__.py           # Package initialization & exports
-│   ├── config.py             # Configuration management
-│   ├── ai/                   # 🧠 AI Integration Module (Refactored)
-│   │   ├── __init__.py       # AI module exports & graceful fallbacks
-│   │   ├── agent_core.py     # LangGraph agent setup & DDChecklistAgent
-│   │   ├── agent_nodes.py    # Individual workflow node functions
-│   │   ├── llm_utilities.py  # Batch processing & utility functions
-│   │   └── prompts.py        # AI prompt templates
-│   ├── document_processing.py # Document operations & FAISS integration
-│   ├── services.py           # Business logic services
-│   ├── ui_components.py      # Reusable UI components
-│   └── utils.py              # Error handling & utilities
+├── app/                       # 📦 Main application package
+│   ├── main.py                # 🎯 Main Streamlit application
+│   ├── __init__.py
+│   ├── ai/                    # 🧠 AI Integration Module
+│   │   ├── __init__.py
+│   │   ├── agent_core.py      # LangGraph agent setup & DDChecklistAgent
+│   │   ├── agent_utils.py     # AI utility functions
+│   │   ├── document_classifier.py # Document classification
+│   │   ├── processing_pipeline.py # AI processing workflows
+│   │   └── prompts.py         # AI prompt templates
+│   ├── core/                  # Core functionality
+│   │   ├── __init__.py
+│   │   ├── config.py          # Configuration management
+│   │   ├── constants.py       # Application constants
+│   │   ├── content_ingestion.py # Document ingestion
+│   │   ├── document_processor.py # Document processing
+│   │   ├── exceptions.py      # Custom exceptions
+│   │   ├── logging.py         # Logging configuration
+│   │   ├── model_cache.py     # Model caching system
+│   │   ├── parsers.py         # Data parsers
+│   │   ├── reports.py         # Report generation
+│   │   ├── search.py          # Search functionality
+│   │   └── utils.py           # Utility functions
+│   ├── handlers/              # Request handlers
+│   │   ├── __init__.py
+│   │   ├── ai_handler.py      # AI request handling
+│   │   ├── document_handler.py # Document operations
+│   │   └── export_handler.py  # Export functionality
+│   ├── services/              # Business logic services
+│   │   ├── ai_client.py       # AI client service
+│   │   ├── ai_config.py       # AI configuration
+│   │   ├── ai_service.py      # AI service layer
+│   │   └── response_parser.py # Response parsing
+│   ├── ui/                    # User interface components
+│   │   ├── __init__.py
+│   │   ├── components.py      # UI components
+│   │   ├── sidebar.py         # Sidebar component
+│   │   ├── tabs/              # Tab components
+│   │   │   ├── __init__.py
+│   │   │   ├── checklist_tab.py
+│   │   │   ├── overview_tab.py
+│   │   │   ├── qa_tab.py
+│   │   │   ├── questions_tab.py
+│   │   │   └── strategic_tab.py
+│   │   └── ui_components/     # Additional UI components
+│   ├── error_handler.py       # Error handling
+│   └── session_manager.py     # Session management
 ├── data/                      # 📊 Data directories
 │   ├── checklist/           # Due diligence checklists (.md)
 │   ├── questions/           # Question lists (.md)
 │   ├── strategy/            # Strategic documents (.md)
+│   ├── search_indexes/      # FAISS and BM25 indices with metadata
 │   └── vdrs/               # Virtual Data Rooms (2 projects)
 │       ├── automated-services-transformation/
 │       └── industrial-security-leadership/
-├── Dockerfile                 # 🐳 Docker container configuration
-├── docker-compose.yml         # 🐳 Docker Compose for local testing
-├── .dockerignore             # Docker build optimization
-├── build-and-run.sh          # 🐳 Docker build & run script
-├── requirements.txt           # Python dependencies (for reference)
-├── pyproject.toml            # uv project configuration
-├── run.sh                    # 🚀 Launch script
+├── models/                   # 🤖 Cached AI models
+│   ├── sentence_transformers/
+│   └── cross_encoder/
+├── tests/                    # 🧪 Test suite
+│   ├── unit/                # Unit tests
+│   ├── integration/         # Integration tests
+│   └── conftest.py          # Test configuration
+├── pyproject.toml            # Python dependencies and project configuration
+├── scripts/start.py          # 🚀 Launch script (Python)
+├── uv.lock                   # uv dependency lock file
 ├── .env                      # API keys (create this)
-├── .venv/                    # uv virtual environment (auto-created)
-└── .logs/                   # Application logs (auto-created)
+└── README.md                 # This file
 ```
 
 ## 🎨 Key Features Explained
@@ -267,7 +568,7 @@ dd_poc/
 - **Supported Formats**: PDF, DOCX, DOC, TXT, MD
 - **Parallel Processing**: Multi-threaded document extraction (4 workers default)
 - **Smart Chunking**: 400-character chunks with 50-character overlap
-- **Embeddings**: Sentence-transformers (all-MiniLM-L6-v2, 384 dimensions)
+- **Embeddings**: Sentence-transformers (all-mpnet-base-v2, 768 dimensions)
 - **Vector Store**: FAISS IndexFlatIP for 10x faster similarity search
 - **Caching**: Intelligent embedding cache with invalidation
 
@@ -315,21 +616,31 @@ dd_poc/
 4. Add ANTHROPIC_API_KEY in Streamlit secrets
 5. Deploy (automatic)
 
-### Option 2: Docker (Production Ready)
+## 🤖 Model Caching for Streamlit Cloud
+
+To optimize performance and avoid download delays on Streamlit Cloud, models are cached locally in the repository:
+
+### Download Models Locally
 ```bash
-# Quick start with Docker
-./build-and-run.sh
+# Download and cache models for offline use
+python download_models.py
+```
 
-# Or manually
-docker build -t dd-checklist .
-docker run -d -p 8501:8501 --name dd-checklist-app dd-checklist
+### Cached Models
+- **Sentence Transformer**: `sentence-transformers/all-mpnet-base-v2` (~418MB)
+- **Cross-Encoder**: `cross-encoder/ms-marco-MiniLM-L-6-v2` (~88MB)
 
-# Using docker-compose
-docker-compose up --build
+### Automatic Model Loading
+The application automatically:
+1. Checks for local models in `models/` directory first
+2. Falls back to HuggingFace download if local models not found
+3. Caches loaded models in memory for reuse
 
-# Stop container
-docker stop dd-checklist-app
-```
+### Benefits
+- ⚡ **Faster startup**: No download delays on Streamlit Cloud
+- 💾 **Offline capable**: Works without internet for model loading
+- 🔄 **Version control**: Models are versioned with your code
+- 🚀 **Consistent performance**: Same model versions across deployments
 
 ### Option 3: Local Development
 ```bash
@@ -337,7 +648,7 @@ docker stop dd-checklist-app
 uv sync
 
 # Run with hot reload for development
-uv run streamlit run app.py --server.runOnSave true
+uv run streamlit run app/main.py --server.runOnSave true
 
 # Add new dependencies
 uv add <package-name>
@@ -346,12 +657,6 @@ uv add <package-name>
 uv lock --upgrade
 ```
 
-### Docker Features
-- **Multi-stage build** for optimized image size
-- **Security-focused** with non-root user
-- **Health checks** for load balancers
-- **Volume mounts** for data persistence
-- **Production ready** with proper environment configuration
 
 ## 💡 Usage Tips
 
@@ -437,10 +742,10 @@ batch_size: int = 100
 uv run python -c "from app import DDChecklistApp; app = DDChecklistApp(); print('✅ App working')"
 
 # Test AI module specifically
-uv run python -c "from src.ai import DDChecklistAgent, LANGGRAPH_AVAILABLE; print('✅ AI available:', LANGGRAPH_AVAILABLE)"
+uv run python -c "from app.ai import agent_core; print('✅ AI module available')"
 
 # Check project structure
-ls -la src/ && ls -la src/ai/
+ls -la app/ && ls -la app/ai/
 
 # Clean Python cache files
 find . -name "*.pyc" -delete && find . -name "__pycache__" -type d -exec rm -rf {} + 2>/dev/null || true
@@ -487,21 +792,33 @@ MIT License - See LICENSE file for details
 
 This application uses a **modular architecture** with clear separation of concerns:
 
-- **`app.py`**: Main Streamlit application orchestrator
-- **`src/`**: All modules organized by responsibility
-  - **`config.py`**: Configuration management with dataclasses
-  - **`ai/`**: **AI Integration Module** (newly refactored)
+- **`app/main.py`**: Main Streamlit application orchestrator
+- **`app/`**: All modules organized by responsibility
+  - **`core/`**: Core functionality
+    - **`config.py`**: Configuration management with dataclasses
+    - **`document_processor.py`**: File handling, text extraction, and FAISS integration
+    - **`parsers.py`**: Data parsing and processing
+    - **`search.py`**: Search functionality with FAISS integration
+    - **`utils.py`**: Error handling, logging, and utilities
+  - **`ai/`**: **AI Integration Module**
     - **`agent_core.py`**: LangGraph agent setup & DDChecklistAgent class
-    - **`agent_nodes.py`**: Individual workflow node functions  
-    - **`llm_utilities.py`**: Batch processing & utility functions
+    - **`agent_utils.py`**: AI utility functions and helpers
+    - **`processing_pipeline.py`**: AI processing workflows and pipelines
     - **`prompts.py`**: AI prompt templates
-  - **`document_processing.py`**: File handling, text extraction, and FAISS integration
-  - **`services.py`**: Business logic (parsing, matching, Q&A)
-  - **`ui_components.py`**: Reusable Streamlit components
-  - **`utils.py`**: Error handling, logging, and utilities
+  - **`handlers/`**: Request handlers
+    - **`ai_handler.py`**: AI request processing
+    - **`document_handler.py`**: Document operations
+    - **`export_handler.py`**: Export functionality
+  - **`services/`**: Business logic services
+    - **`ai_service.py`**: AI service layer
+    - **`ai_client.py`**: AI client interface
+    - **`response_parser.py`**: Response parsing and formatting
+  - **`ui/`**: User interface components
+    - **`components.py`**: Reusable Streamlit components
+    - **`tabs/`**: Tab-specific UI components
 
 ### Key Architectural Improvements (2025)
-- ✅ **Refactored AI Module**: Broke down 733-line monolith into focused modules
+- ✅ **Modular Design**: Clean separation between core, AI, handlers, services, and UI
 - ✅ **FAISS Integration**: 10x faster document similarity search
 - ✅ **Parallel Processing**: Multi-threaded document extraction
 - ✅ **Current Models**: Updated to 2025 Claude model names
@@ -511,17 +828,17 @@ This application uses a **modular architecture** with clear separation of concer
 ## 🤝 Contributing
 
 Contributions welcome! The modular architecture makes it easy to extend:
-- Add new AI models in `src/ai/agent_core.py`
-- Extend document processing in `src/document_processing.py`
-- Add UI components in `src/ui_components.py`
-- Create new services in `src/services.py`
+- Add new AI models in `app/ai/agent_core.py`
+- Extend document processing in `app/core/document_processor.py`
+- Add UI components in `app/ui/components.py`
+- Create new services in `app/services/`
 
 ## 📧 Support
 
 For questions or support:
 1. Check the [troubleshooting section](#-troubleshooting)
-2. Test your setup: `uv run python -c "from app import DDChecklistApp; from src.ai import DDChecklistAgent; print('✅ Ready')"`
-3. Verify AI models: `uv run python -c "from src.ai import DDChecklistAgent; agent = DDChecklistAgent(); print('✅ AI available:', agent.is_available())"`
+2. Test your setup: `uv run python -c "from app import main; print('✅ App ready')"`
+3. Verify AI models: `uv run python -c "from app.ai.agent_core import DDChecklistAgent; print('✅ AI available')"`
 4. Open an issue on GitHub
 
 ---

app.py

@@ -1,599 +0,0 @@
-#!/usr/bin/env python3
-"""
-DD-Checklist Main Application - Refactored Version
-
-This is the main Streamlit application that orchestrates all components
-using the new modular architecture for better maintainability.
-"""
-
-import os
-import warnings
-import logging
-
-# Fix tokenizers parallelism warning early
-os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
-
-# Only suppress specific known non-critical warnings
-warnings.filterwarnings("ignore", message=".*Relevance scores must be between.*")
-warnings.filterwarnings("ignore", message=".*No relevant docs were retrieved.*")
-
-import streamlit as st
-
-from pathlib import Path
-from typing import Dict
-
-# Import our refactored modules
-from src import (
-    init_config, DocumentProcessor,
-    logger,
-    render_project_selector,
-    render_ai_settings, escape_markdown_math,
-    get_mime_type, format_document_title
-)
-from src.config import configure_langchain_logging
-from src.document_processing import safe_execute
-# Using Streamlit directly for simplicity
-from src.ui_components import (
-    render_file_selector, render_checklist_results, render_question_results,
-    render_quick_questions, create_document_link
-)
-from src.services import (
-    search_documents
-)
-
-from src.config import show_success, show_error, show_info
-
-# Import LangGraph + Anthropic configuration
-from src.ai import (
-    DDChecklistAgent
-)
-
-
-class DDChecklistApp:
-    """
-    Main application class that orchestrates all components
-    """
-
-    def __init__(self):
-        """Initialize the application"""
-        # Initialize configuration
-        self.config = init_config()
-        
-        # Initialize session state
-        self._init_session_state()
-        
-        # Configure Streamlit page
-        st.set_page_config(
-            page_title=self.config.ui.page_title,
-            page_icon=self.config.ui.page_icon,
-            layout=self.config.ui.layout
-        )
-        
-        # Initialize services (will be loaded when needed)
-        self.model_name = self.config.model.sentence_transformer_model
-        self.document_processor = None
-        self.agent = None
-    
-    def _init_session_state(self):
-        """Initialize essential session state variables only"""
-        essential_defaults = {
-            'documents': {},
-            'chunks': [],
-            'embeddings': None,
-            'checklist_results': {},
-            'question_answers': {},
-            'company_summary': "",
-            'strategy_analysis': "",
-            'agent': None,
-            # Sidebar file selections
-            'selected_strategy_path': None,
-            'selected_strategy_text': "",
-            'selected_checklist_path': None,
-            'selected_checklist_text': "",
-            'selected_questions_path': None,
-            'selected_questions_text': ""
-        }
-        
-        for key, default_value in essential_defaults.items():
-            if key not in st.session_state:
-                st.session_state[key] = default_value
-
-    def initialize_services(self):
-        """Initialize core services"""
-        if self.document_processor is None:
-            self.document_processor = DocumentProcessor(self.model_name)
-            
-            # Restore document processor state from session state if available
-            if (hasattr(st.session_state, 'chunks') and st.session_state.chunks and
-                hasattr(st.session_state, 'embeddings') and st.session_state.embeddings is not None):
-                
-                self.document_processor.chunks = st.session_state.chunks
-                self.document_processor.embeddings = st.session_state.embeddings
-                # Note: Don't restore documents here - they'll be recreated from chunks if needed
-    
-    def setup_ai_agent(self, api_key: str, model_choice: str) -> bool:
-        """
-        Setup AI agent
-        
-        Args:
-            api_key: Anthropic API key
-            model_choice: Claude model to use
-            
-        Returns:
-            True if agent was successfully initialized
-        """        
-        try:
-            with st.spinner("Initializing AI agent..."):
-                agent = DDChecklistAgent(api_key, model_choice)
-                
-                if agent.is_available():
-                    st.session_state.agent = agent
-                    self.agent = agent
-                    show_success("✅ AI Agent ready")
-                    
-                    
-                    return True
-                else:
-                    show_error("❌ Failed to initialize agent")
-                    return False
-        except Exception as e:
-            show_error(f"Agent initialization failed: {str(e)}")
-            return False
-    
-    def render_sidebar(self) -> tuple:
-        """
-        Render sidebar with project selection, file selectors, and AI settings
-        
-        Returns:
-            Tuple of (selected_data_room_path, use_ai_features, process_button)
-        """
-        with st.sidebar:
-            # Project and data room selection
-            selected_project_path, selected_data_room_path = render_project_selector()
-            
-            # Process button
-            process_button = st.button(
-                "🚀 Process Data Room", 
-                type="primary", 
-                use_container_width=True
-            )
-            
-            if process_button:
-                show_success("Processing... Check main area for progress")
-            
-            st.divider()
-            
-            # Strategy, Checklist, and Questions selectors
-            st.subheader("📋 Analysis Configuration")
-            
-            # Strategy selector
-            strategy_path, strategy_text = render_file_selector(
-                self.config.paths.strategy_dir, "Strategy", "sidebar", "🎯"
-            )
-            # Store in session state
-            st.session_state.selected_strategy_path = strategy_path
-            st.session_state.selected_strategy_text = strategy_text
-            
-            # Checklist selector
-            checklist_path, checklist_text = render_file_selector(
-                self.config.paths.checklist_dir, "Checklist", "sidebar", "📊"
-            )
-            # Store in session state
-            st.session_state.selected_checklist_path = checklist_path
-            st.session_state.selected_checklist_text = checklist_text
-            
-            # Questions selector
-            questions_path, questions_text = render_file_selector(
-                self.config.paths.questions_dir, "Questions", "sidebar", "❓"
-            )
-            # Store in session state
-            st.session_state.selected_questions_path = questions_path
-            st.session_state.selected_questions_text = questions_text
-            
-            st.divider()
-            
-            # AI settings
-            use_ai_features, api_key, model_choice = render_ai_settings()
-            
-            # Initialize AI agent if enabled
-            if use_ai_features and api_key:
-                if not hasattr(st.session_state, 'agent') or st.session_state.agent is None:
-                    self.setup_ai_agent(api_key, model_choice)
-                elif hasattr(st.session_state, 'agent') and st.session_state.agent:
-                    self.agent = st.session_state.agent
-            else:
-                st.session_state.agent = None
-                self.agent = None
-        
-        return selected_data_room_path, use_ai_features, process_button
-
-    def render_company_overview_tab(self):
-        """Render company overview tab"""
-        # Use strategy from sidebar
-        strategy_text = st.session_state.get('selected_strategy_text', "")
-        
-        # Check if we have documents to display summaries
-        if st.session_state.documents:
-            self._render_report_section("overview", strategy_text=strategy_text)
-        else:
-            show_info("👈 Configure and process data room to see analysis")
-    
-    def render_strategic_analysis_tab(self):
-        """Render strategic analysis tab"""
-        # Use strategy from sidebar
-        strategy_text = st.session_state.get('selected_strategy_text', "")
-        
-        # Check if we have documents to display summaries
-        if st.session_state.documents:
-            self._render_report_section("strategic", strategy_text=strategy_text)
-        else:
-            show_info("👈 Configure and process data room to see analysis")
-    
-    def _render_report_section(self, report_type: str, strategy_text: str = ""):
-        """Unified report rendering for both overview and strategic analysis"""
-        from src.services import generate_reports
-        
-        summary_key = f"{report_type}_summary"
-        
-        # Check prerequisites for strategic analysis
-        if report_type == "strategic" and not st.session_state.checklist_results:
-            st.warning("⚠️ Process data room with checklist first to enable strategic analysis")
-            return
-        
-        # Auto-generate report if not already present and AI is available
-        if (not st.session_state.get(summary_key, "") and st.session_state.agent):
-            with st.spinner(f"🤖 Generating {report_type} analysis..."):
-                data_room_name = (Path(list(st.session_state.documents.keys())[0]).parent.name 
-                                if st.session_state.documents else "Unknown")
-                
-                st.session_state[summary_key] = generate_reports(
-                    st.session_state.documents,
-                    data_room_name,
-                    strategy_text,
-                    st.session_state.checklist_results,
-                    report_type,
-                    st.session_state.agent.llm if st.session_state.agent else None
-                )
-        
-        # Display the report if available
-        if st.session_state.get(summary_key, ""):
-            st.markdown(st.session_state[summary_key])
-            
-            # Add export and regenerate buttons
-            self._render_report_actions(report_type, summary_key)
-    
-    def _render_report_actions(self, report_type: str, summary_key: str):
-        """Render export and regenerate actions for reports"""
-        if report_type == "overview":
-            col1, col2 = st.columns([1, 5])
-            with col1:
-                company_name = (Path(list(st.session_state.documents.keys())[0]).parent.name 
-                               if st.session_state.documents else 'export')
-                file_name = f"company_overview_{company_name}.md"
-                st.download_button(
-                    "📥 Export Summary",
-                    data=f"# Company Overview\n\n{st.session_state[summary_key]}",
-                    file_name=file_name,
-                    mime="text/markdown",
-                    key=f"export_{summary_key}"
-                )
-            with col2:
-                if st.button(f"🔄 Regenerate {report_type.title()}"):
-                    st.session_state[summary_key] = ""
-                    st.rerun()
-        else:
-            col1, col2 = st.columns([1, 5])
-            with col1:
-                # Combined report export for strategic analysis
-                combined_report = f"# Due Diligence Report\n\n"
-                combined_report += f"## Company Overview\n\n{st.session_state.get('overview_summary', '')}\n\n"
-                combined_report += f"## Strategic Analysis\n\n{st.session_state[summary_key]}"
-                
-                company_name = (Path(list(st.session_state.documents.keys())[0]).parent.name 
-                               if st.session_state.documents else 'export')
-                file_name = f"dd_report_{company_name}.md"
-                st.download_button(
-                    "📥 Export Report",
-                    data=combined_report,
-                    file_name=file_name,
-                    mime="text/markdown",
-                    key=f"export_combined_{summary_key}"
-                )
-            with col2:
-                if st.button(f"🔄 Regenerate {report_type.title()}"):
-                    st.session_state[summary_key] = ""
-                    st.rerun()
-    
-    def render_analysis_tab(self, tab_type: str):
-        """Unified rendering for checklist and questions tabs"""
-        if tab_type == "checklist":
-            # Use checklist from sidebar
-            file_text = st.session_state.get('selected_checklist_text', "")
-            
-            if not file_text:
-                show_info("👈 Select a checklist in the sidebar to see analysis results")
-                return
-            
-            # Render results if available
-            render_checklist_results(st.session_state.checklist_results)
-            
-        elif tab_type == "questions":
-            # Use questions from sidebar
-            file_text = st.session_state.get('selected_questions_text', "")
-            
-            if not file_text:
-                show_info("👈 Select a questions list in the sidebar to see analysis results")
-                return
-            
-            # Render results if available
-            render_question_results(st.session_state.question_answers)
-    
-    def render_qa_tab(self):
-        """Render the Q&A with citations tab"""
-        if not st.session_state.chunks:
-            show_info("👈 Process data room first to enable Q&A")
-            return
-        
-        # Question input
-        question = st.text_input(
-            "Ask a question about your documents:",
-            placeholder="e.g., What are the main risks? What is the revenue model? Who are the key customers?"
-        )
-        
-        # Quick question buttons
-        quick_question = render_quick_questions()
-        if quick_question:
-            question = quick_question
-        
-        st.divider()
-        
-        if question:
-            self._handle_qa_query(question)
-    
-    def _handle_qa_query(self, question: str):
-        """Handle Q&A query and display results"""
-        if not self.document_processor:
-            self.initialize_services()
-        
-        # Use lower threshold for Q&A to get more relevant results
-        qa_threshold = 0.25
-        
-        with st.spinner("🔍 Searching documents..."):
-            results = search_documents(
-                self.document_processor,
-                question, 
-                top_k=self.config.ui.top_k_search_results,
-                threshold=qa_threshold
-            )
-        
-        if results:
-            # Use agent to synthesize answer if available
-            if (hasattr(st.session_state, 'agent') and st.session_state.agent and 
-                hasattr(st.session_state.agent, 'llm')):
-                
-                st.markdown("### 🤖 AI Agent's Answer")
-                with st.spinner("Agent analyzing documents..."):
-                    # Convert results to document format for context
-                    context = "\n\n".join([f"From {r['source']}:\n{r['text']}" for r in results[:3]])
-                    # Use LLM directly for more reliable answers
-                    from langchain_core.messages import HumanMessage
-                    prompt = (f"Question: {question}\n\n"
-                             f"Relevant document excerpts:\n{context}\n\n"
-                             f"Provide a comprehensive answer with citations to the sources.")
-                    response = st.session_state.agent.llm.invoke([HumanMessage(content=prompt)])
-                    # Clean up any leading whitespace and escape math characters
-                    answer_text = escape_markdown_math(response.content.strip())
-                    st.markdown(answer_text)
-                st.divider()
-            
-            st.markdown("### 📚 Source Documents")
-            
-            # Display source documents with download buttons
-            for i, result in enumerate(results[:3], 1):
-                with st.container():
-                    col1, col2 = st.columns([5, 1])
-                    with col1:
-                        excerpt = result['text'][:200] + "..." if len(result['text']) > 200 else result['text']
-                        st.markdown(f"{i}. \"{excerpt}\"")
-                        
-                        # Create clickable link for the document
-                        doc_path = result.get('path', result.get('full_path', ''))
-                        doc_name = result['source']
-                        doc_title = format_document_title(doc_name)
-                        
-                        if doc_path:
-                            # Create unique key for this result
-                            unique_key = f"result_{i}_{hash(doc_path) % 10000}"
-                            col_a, col_b = st.columns([3, 1])
-                            with col_a:
-                                create_document_link(doc_path, doc_name, doc_title, unique_key)
-                            with col_b:
-                                st.caption(f"({result['citation']})")
-                        else:
-                            st.caption(f"   📄 {result['source']} ({result['citation']})")
-                    
-                    with col2:
-                        self._render_qa_download_button(result, i, question)
-        else:
-            st.warning("No relevant information found for your question.")
-    
-    def _render_qa_download_button(self, result: Dict, idx: int, question: str):
-        """Render download button for Q&A results"""
-        doc_path = result.get('path', '')
-        if doc_path:
-            try:
-                file_path = Path(doc_path)
-                if not file_path.is_absolute():
-                    file_path = Path("data") / file_path
-                
-                if file_path.exists():
-                    with open(file_path, 'rb') as f:
-                        file_bytes = f.read()
-                    
-                    # Determine MIME type based on file extension
-                    mime_type = get_mime_type(file_path)
-                    
-                    button_key = f"qacit_dl_{idx}_{question[:20]}".replace(" ", "_").replace("?", "")
-                    
-                    st.download_button(
-                        label="📥 Download",
-                        data=file_bytes,
-                        file_name=result['source'],
-                        mime=mime_type,
-                        key=button_key,
-                        help=f"Download {result['source']}"
-                    )
-            except Exception as e:
-                st.error(f"Download failed: {str(e)}")
-    
-    def process_data_room(self, data_room_path: str):
-        """Simplified data room processing"""
-        if not Path(data_room_path).exists():
-            show_error(f"Data room path not found: {data_room_path}")
-            return
-        
-        # Use safe_execute for the entire processing operation
-        def process_operation():
-            self.initialize_services()
-            # Simple processing - load documents
-            self.document_processor.load_data_room(data_room_path)
-            
-            # Store results in session state with simplified structure
-            # Convert list of LangChain documents to dictionary format expected by UI
-            documents_dict = {}
-            for doc in self.document_processor.documents:
-                file_path = doc.metadata.get('source', doc.metadata.get('path', 'unknown'))
-                documents_dict[file_path] = {
-                    'name': doc.metadata.get('name', Path(file_path).name if file_path != 'unknown' else 'unknown'),
-                    'path': doc.metadata.get('path', ''),
-                    'content': doc.page_content,
-                    'metadata': doc.metadata
-                }
-            
-            st.session_state.documents = documents_dict
-            st.session_state.chunks = self.document_processor.chunks
-            st.session_state.embeddings = self.document_processor.embeddings
-            
-            # Process checklist and questions if available
-            self._process_checklist_and_questions()
-            
-            # Clear any existing analysis to trigger regeneration
-            st.session_state.company_summary = ""
-            st.session_state.strategy_analysis = ""
-            st.session_state.overview_summary = ""
-            st.session_state.strategic_summary = ""
-            
-            show_success("✅ Data room processing complete! View results in the tabs above.")
-            st.rerun()
-            
-        safe_execute(
-            process_operation,
-            None,
-            "Data room processing"
-        )
-    
-    def _process_checklist_and_questions(self):
-        """Process checklist and questions after documents are loaded"""
-        from src.services import parse_checklist, parse_questions, create_vector_store, search_and_analyze
-        
-        # Use checklist from sidebar selection
-        checklist_text = st.session_state.get('selected_checklist_text', "")
-        if checklist_text and self.document_processor.chunks:
-            try:
-                # Parse checklist
-                checklist = parse_checklist(checklist_text)
-                st.session_state.checklist = checklist
-                
-                # Create vector store from chunks for processing
-                vector_store = create_vector_store(self.document_processor.chunks, self.model_name)
-                
-                # Process checklist items
-                checklist_results = search_and_analyze(
-                    checklist,
-                    vector_store,
-                    self.agent.llm if self.agent else None,
-                    self.config.processing.similarity_threshold,
-                    'items'
-                )
-                st.session_state.checklist_results = checklist_results
-                logger.info("✅ Checklist processing completed")
-            except Exception as e:
-                logger.error(f"Checklist processing failed: {e}")
-        
-        # Use questions from sidebar selection
-        questions_text = st.session_state.get('selected_questions_text', "")
-        if questions_text and self.document_processor.chunks:
-            try:
-                # Parse questions
-                questions = parse_questions(questions_text)
-                st.session_state.questions = questions
-                
-                # Create vector store from chunks for processing (reuse if already created)
-                if 'vector_store' not in locals():
-                    vector_store = create_vector_store(self.document_processor.chunks, self.model_name)
-                
-                # Process questions
-                question_answers = search_and_analyze(
-                    questions,
-                    vector_store,
-                    self.agent.llm if self.agent else None,
-                    self.config.processing.relevancy_threshold,
-                    'questions'
-                )
-                st.session_state.question_answers = question_answers
-                logger.info("✅ Questions processing completed")
-            except Exception as e:
-                logger.error(f"Questions processing failed: {e}")
-    
-    def run(self):
-        """Run the main application"""
-        # Render header
-        st.title("🤖 AI Due Diligence")
-        st.markdown("**Intelligent M&A Analysis:** Strategic assessment, automated document review, and AI-powered insights")
-        
-        # Render sidebar and get selections
-        selected_data_room_path, use_ai_features, process_button = self.render_sidebar()
-        
-        # Main tabs - Company Overview and Strategic Analysis moved to top level
-        tab1, tab2, tab3, tab4, tab5 = st.tabs([
-            "🏢 Company Overview",
-            "🎯 Strategic Analysis", 
-            "📊 Checklist Matching", 
-            "❓ Due Diligence Questions", 
-            "💬 Q&A with Citations"
-        ])
-        
-        with tab1:
-            self.render_company_overview_tab()
-        
-        with tab2:
-            self.render_strategic_analysis_tab()
-        
-        with tab3:
-            self.render_analysis_tab("checklist")
-        
-        with tab4:
-            self.render_analysis_tab("questions")
-        
-        with tab5:
-            self.render_qa_tab()
-        
-        # Processing complete message is handled in process_data_room function
-        
-        # Simplified processing trigger
-        if process_button and selected_data_room_path:
-            with st.spinner("🚀 Processing data room..."):
-                self.process_data_room(selected_data_room_path)
-
-
-def main():
-    """Main application entry point"""
-    # Configure LangChain logging to reduce verbosity  
-    configure_langchain_logging(log_level="WARNING")
-    
-    app = DDChecklistApp()
-    app.run()
-
-
-if __name__ == "__main__":
-    main()

app/__init__.py

@@ -0,0 +1,7 @@
+"""
+Refactored DD Checklist Application
+
+A modular Streamlit application for AI-powered due diligence analysis.
+"""
+
+__version__ = "2.0.0"

{src → app}/ai/__init__.py

@@ -18,27 +18,27 @@ from .prompts import (
 
 # Direct imports for AI functionality - assuming dependencies are present
 from .agent_core import (
-    DDChecklistAgent, 
-    get_langgraph_agent,
-    AgentState, 
+    Agent,
+    get_langgraph_agent
+)
+from .agent_utils import (
+    AgentState,
     TaskType
 )
 
 # Export main public API
 __all__ = [
     # Core agent functionality
-    'DDChecklistAgent',
+    'Agent',
     'get_langgraph_agent',
-    
 
-    
-    # Agent types and state (now in agent_core)
+    # Agent types and state
     'AgentState',
     'TaskType',
-    
+
     # Prompt functions
     'get_checklist_parsing_prompt',
-    'get_document_relevance_prompt', 
+    'get_document_relevance_prompt',
     'get_question_answering_prompt',
     'get_findings_summary_prompt',
     'get_description_generation_prompt',

app/ai/agent_core.py

@@ -0,0 +1,277 @@
+#!/usr/bin/env python3
+"""
+LangGraph Agent Core Module
+
+This module contains the main LangGraph agent setup and the high-level
+Agent class for interacting with the agent system.
+"""
+
+# Standard library imports
+import logging
+from typing import Optional, Dict, List, Any, Tuple
+
+# Third-party imports
+import streamlit as st
+from langchain_anthropic import ChatAnthropic
+from langchain_core.messages import HumanMessage, AIMessage
+from langgraph.checkpoint.memory import MemorySaver
+from langgraph.graph import StateGraph, END
+
+# Local imports
+from app.ai.agent_utils import AgentState
+from app.ai.processing_pipeline import route_task, route_condition
+from app.ai.processing_pipeline import (
+    parse_checklist_node,
+    match_checklist_node,
+    answer_question_node,
+    summarize_node
+)
+from app.core.config import get_config
+
+logger = logging.getLogger(__name__)
+
+
+
+# Agent Functions
+
+def get_langgraph_agent(api_key: Optional[str] = None, model: Optional[str] = None) -> Optional[Tuple[Any, "ChatAnthropic"]]:
+    """
+    Create a LangGraph agent with Anthropic
+    
+    Args:
+        api_key: Anthropic API key (optional, will be sourced from environment/config)
+        model: Model name to use (optional, will use config default)
+        
+    Returns:
+        Tuple of (compiled_app, llm) or None if not available
+    """
+    
+    # Get configuration
+    config = get_config()
+    
+    # Get API key from various sources
+    if not api_key:
+        api_key = config.api.anthropic_api_key
+        if not api_key and st and hasattr(st, 'secrets') and 'ANTHROPIC_API_KEY' in st.secrets:
+            api_key = st.secrets['ANTHROPIC_API_KEY']
+    
+    if not api_key:
+        return None
+    
+    # Use model from config if not specified
+    if not model:
+        model = config.model.claude_model
+    
+    # Initialize Claude with config values
+    llm = ChatAnthropic(
+        model=model,
+        anthropic_api_key=api_key,
+        temperature=config.model.temperature,
+        max_tokens=config.model.max_tokens
+    )
+    
+    # No custom tools needed - using built-in LangGraph functionality
+    
+    # Create the graph
+    workflow = StateGraph(AgentState)
+    
+    # Create node functions that have access to the llm
+    def _route_task(state: AgentState) -> AgentState:
+        return route_task(state)
+
+    def _parse_checklist_node(state: AgentState) -> AgentState:
+        return parse_checklist_node(state, llm)
+
+    def _match_checklist_node(state: AgentState) -> AgentState:
+        return match_checklist_node(state, llm)
+
+    def _answer_question_node(state: AgentState) -> AgentState:
+        return answer_question_node(state, llm)
+
+    def _summarize_node(state: AgentState) -> AgentState:
+        return summarize_node(state, llm)
+    
+    # Add nodes to workflow
+    workflow.add_node("route", _route_task)
+    workflow.add_node("parse_checklist", _parse_checklist_node)
+    workflow.add_node("match_checklist", _match_checklist_node)
+    workflow.add_node("answer_question", _answer_question_node)
+    workflow.add_node("summarize", _summarize_node)
+    
+    # Define edges
+    workflow.set_entry_point("route")
+    
+    # Conditional routing based on next_action
+    workflow.add_conditional_edges(
+        "route",
+        route_condition,
+        {
+            "parse_checklist": "parse_checklist",
+            "match_checklist": "match_checklist",
+            "answer_question": "answer_question",
+            "summarize": "summarize"
+        }
+    )
+    
+    # All task nodes go to END
+    workflow.add_edge("parse_checklist", END)
+    workflow.add_edge("match_checklist", END)
+    workflow.add_edge("answer_question", END)
+    workflow.add_edge("summarize", END)
+    
+    # Compile with memory
+    memory = MemorySaver()
+    app = workflow.compile(checkpointer=memory)
+    
+    return app, llm
+
+
+class Agent:
+    """High-level interface for the LangGraph agent"""
+    
+    def __init__(self, api_key: Optional[str] = None, model: Optional[str] = None):
+        """
+        Initialize the Agent
+
+        Args:
+            api_key: Anthropic API key (optional)
+            model: Model name to use
+        """
+        result = get_langgraph_agent(api_key, model)
+        if result:
+            self.app, self.llm = result
+            self.thread_id = "dd-poc-session"
+        else:
+            self.app = None
+            self.llm = None
+    
+    def is_available(self) -> bool:
+        """Check if the agent is available for use"""
+        return self.app is not None and self.llm is not None
+    
+    def parse_checklist(self, checklist_text: str) -> Optional[Dict]:
+        """
+        Parse checklist using the agent
+        
+        Args:
+            checklist_text: Raw checklist text to parse
+            
+        Returns:
+            Parsed checklist dictionary or None if failed
+        """
+        if not self.app:
+            return None
+        
+        try:
+            # Run the agent
+            result = self.app.invoke(
+                {"messages": [HumanMessage(content=f"Parse this checklist: {checklist_text}")]},
+                config={"configurable": {"thread_id": self.thread_id}}
+            )
+            
+            return result.get("checklist")
+        except Exception as e:
+            st.error(f"Agent error: {str(e)}")
+            return None
+    
+    def match_documents(self, checklist: Dict, documents: List[Dict]) -> Dict:
+        """
+        Match documents to checklist items
+        
+        Args:
+            checklist: Parsed checklist dictionary
+            documents: List of document dictionaries
+            
+        Returns:
+            Dictionary of findings or empty dict if failed
+        """
+        if not self.app:
+            return {}
+        
+        try:
+            # Prepare state
+            initial_state = {
+                "messages": [HumanMessage(content="Match documents to checklist items")],
+                "checklist": checklist,
+                "documents": documents,
+                "findings": {}
+            }
+            
+            result = self.app.invoke(
+                initial_state,
+                config={"configurable": {"thread_id": self.thread_id}}
+            )
+            
+            return result.get("findings", {})
+        except Exception as e:
+            st.error(f"Agent error: {str(e)}")
+            return {}
+    
+    def answer_question(self, question: str, documents: List[Dict]) -> str:
+        """
+        Answer a question using document context
+        
+        Args:
+            question: User question
+            documents: List of document dictionaries for context
+            
+        Returns:
+            Answer string or error message
+        """
+        if not self.app:
+            return "Agent not available"
+        
+        try:
+            initial_state = {
+                "messages": [HumanMessage(content=question)],
+                "documents": documents
+            }
+            
+            result = self.app.invoke(
+                initial_state,
+                config={"configurable": {"thread_id": self.thread_id}}
+            )
+            
+            # Get the last AI message
+            messages = result.get("messages", [])
+            for msg in reversed(messages):
+                if isinstance(msg, AIMessage):
+                    return msg.content
+            
+            return "No answer generated"
+        except Exception as e:
+            return f"Error: {str(e)}"
+    
+    def summarize_findings(self, findings: Dict) -> str:
+        """
+        Generate executive summary
+        
+        Args:
+            findings: Dictionary of due diligence findings
+            
+        Returns:
+            Summary string or error message
+        """
+        if not self.app:
+            return "Agent not available"
+        
+        try:
+            initial_state = {
+                "messages": [HumanMessage(content="Summarize the due diligence findings")],
+                "findings": findings
+            }
+            
+            result = self.app.invoke(
+                initial_state,
+                config={"configurable": {"thread_id": self.thread_id}}
+            )
+            
+            # Get the last AI message
+            messages = result.get("messages", [])
+            for msg in reversed(messages):
+                if isinstance(msg, AIMessage):
+                    return msg.content
+            
+            return "No summary generated"
+        except Exception as e:
+            return f"Error: {str(e)}"

app/ai/agent_utils.py

@@ -0,0 +1,150 @@
+#!/usr/bin/env python3
+"""
+Agent Utilities Module
+
+This module contains utility functions, helper methods, and type definitions
+for the LangGraph agent system.
+"""
+
+# Standard library imports
+import logging
+import random
+import time
+from enum import Enum
+from typing import Optional, Dict, List, Sequence
+
+# Third-party imports
+from langchain_core.runnables import RunnableLambda
+from typing_extensions import TypedDict
+
+# Local imports
+from app.core.config import get_config
+
+logger = logging.getLogger(__name__)
+
+
+def with_retry(func, max_attempts=3, base_delay=1.0):
+    """
+    Wrapper function to add exponential backoff retry logic to any function.
+
+    Args:
+        func: Function to wrap with retry logic
+        max_attempts: Maximum number of retry attempts (default: 3)
+        base_delay: Base delay in seconds for exponential backoff (default: 1.0)
+
+    Returns:
+        Wrapped function with retry logic
+    """
+    def wrapper(*args, **kwargs):
+        for attempt in range(max_attempts):
+            try:
+                return func(*args, **kwargs)
+            except Exception as e:
+                if attempt == max_attempts - 1:  # Last attempt
+                    raise e
+
+                # Exponential backoff with jitter
+                delay = base_delay * (2 ** attempt) + random.uniform(0, 1)
+                logger.warning(f"Attempt {attempt + 1} failed: {str(e)}. Retrying in {delay:.2f}s...")
+                time.sleep(delay)
+
+    return wrapper
+
+
+def create_batch_processor(llm: "ChatAnthropic", max_concurrency: int = None) -> RunnableLambda:
+    """
+    Create a batch processor using LangChain's retry and fallback mechanisms.
+
+    Args:
+        llm: ChatAnthropic instance
+        max_concurrency: Maximum concurrent requests (uses config default if None)
+
+    Returns:
+        RunnableLambda configured with retry and fallback mechanisms
+    """
+    config = get_config()
+    if max_concurrency is None:
+        max_concurrency = 3  # Default max concurrency
+
+    def process_single_item(input_data):
+        """Process a single item with error handling"""
+        try:
+            messages, item_info = input_data
+            response = llm.invoke(messages)
+            return {
+                'success': True,
+                'response': response,
+                'item_info': item_info,
+                'error': None
+            }
+        except Exception as e:
+            # Fail immediately on any error
+            error_msg = f"Single item processing failed: {str(e)}"
+            logger.error(error_msg)
+            raise Exception(error_msg)
+
+    def process_batch(batch_inputs):
+        """Process a batch of inputs with individual item error handling"""
+        try:
+            # Use LLM's batch method for efficiency
+            messages_batch = [input_data[0] for input_data in batch_inputs]
+            item_infos = [input_data[1] for input_data in batch_inputs]
+
+            responses = llm.batch(
+                messages_batch,
+                config={"max_concurrency": max_concurrency}
+            )
+
+            # Process results with individual error handling - fail on any error
+            results = []
+            for i, (response, item_info) in enumerate(zip(responses, item_infos)):
+                if response:
+                    results.append({
+                        'success': True,
+                        'response': response,
+                        'item_info': item_info,
+                        'error': None
+                    })
+                else:
+                    # Fail immediately on any missing response
+                    error_msg = f'No response for item {i}'
+                    logger.error(error_msg)
+                    raise Exception(error_msg)
+
+            return results
+
+        except Exception as e:
+            # If batch fails completely, fail immediately
+            error_msg = f"Batch processing failed: {e}"
+            logger.error(error_msg)
+            raise Exception(error_msg)
+
+    # Create the main processor with retry logic
+    retryable_process_batch = with_retry(process_batch, max_attempts=3, base_delay=1.0)
+    processor = RunnableLambda(retryable_process_batch)
+
+    return processor
+
+
+# =============================================================================
+# TYPE DEFINITIONS
+# =============================================================================
+
+# Define the state for our agent
+class AgentState(TypedDict):
+    """State for the due diligence agent"""
+    messages: Sequence["BaseMessage"]
+    checklist: Optional[Dict]
+    documents: Optional[List[Dict]]
+    current_task: Optional[str]
+    findings: Dict[str, List[str]]
+    next_action: Optional[str]
+
+
+class TaskType(Enum):
+    """Types of tasks the agent can perform"""
+    PARSE_CHECKLIST = "parse_checklist"
+    ANALYZE_DOCUMENT = "analyze_document"
+    MATCH_CHECKLIST = "match_checklist"
+    ANSWER_QUESTION = "answer_question"
+    SUMMARIZE_FINDINGS = "summarize_findings"

app/ai/document_classifier.py

@@ -0,0 +1,140 @@
+#!/usr/bin/env python3
+"""
+Document Classification Module
+
+This module contains functions for classifying document types and related utilities.
+"""
+
+# Standard library imports
+import logging
+from typing import List, Dict, Optional
+
+# Third-party imports
+from langchain_core.messages import HumanMessage
+import httpx
+import backoff
+
+# Local imports
+from app.ai.agent_utils import create_batch_processor
+from app.ai.prompts import get_document_type_classification_prompt
+from app.core.config import get_config
+from app.core.constants import DEFAULT_BATCH_SIZE
+from app.core.performance import get_performance_manager
+
+logger = logging.getLogger(__name__)
+
+
+@backoff.on_exception(
+    backoff.expo,
+    (Exception,),
+    max_tries=3,
+    jitter=backoff.random_jitter
+)
+def batch_classify_document_types(first_chunks: List[Dict], llm: "ChatAnthropic", batch_size: Optional[int] = None) -> List[Dict]:
+    """
+    Fast document type classification using first chunks only with Haiku model.
+    Optimized for speed and cost with batched processing.
+
+    Args:
+        first_chunks: List of first chunk dictionaries to classify
+        llm: ChatAnthropic instance (should be Haiku for speed/cost)
+        batch_size: Number of documents to process in each batch (uses config default if None)
+
+    Returns:
+        List of documents with added document_type field
+    """
+    config = get_config()
+    if batch_size is None:
+        # Use optimized batch size for Haiku (faster model)
+        batch_size = min(DEFAULT_BATCH_SIZE, 25)  # Increased to 25 docs per batch for better performance
+
+    # Create batch processor with retry and fallback mechanisms
+    batch_processor = create_batch_processor(llm, max_concurrency=5)  # Increased concurrency
+
+    # Process documents in batches
+    classified_docs = []
+    total_docs = len(first_chunks)
+    total_batches = (total_docs + batch_size - 1) // batch_size
+
+    model_name = getattr(llm, 'model', 'unknown')
+    logger.info(f"🏷️ Classifying {total_docs} document types using {model_name}")
+
+    # Get performance manager for caching
+    perf_manager = get_performance_manager()
+
+    for batch_num, i in enumerate(range(0, total_docs, batch_size), 1):
+        batch = first_chunks[i:i + batch_size]
+        batch_end = min(i + batch_size, total_docs)
+
+        # Check cache for existing classifications
+        cached_batch = []
+        uncached_batch = []
+        uncached_indices = []
+
+        for idx, doc in enumerate(batch):
+            cache_key = f"classification:{doc.get('path', '')}"
+            cached_result = perf_manager.doc_cache.get(cache_key)
+            if cached_result:
+                cached_batch.append(cached_result)
+                logger.debug(f"Cache hit for document classification: {doc.get('name', '')}")
+            else:
+                uncached_batch.append(doc)
+                uncached_indices.append(idx)
+
+        logger.info(f"Processing classification batch {batch_num}/{total_batches} "
+                   f"({len(uncached_batch)} new, {len(cached_batch)} cached documents)")
+
+        # Only process uncached documents
+        if uncached_batch:
+            batch_inputs = []
+            for doc in uncached_batch:
+                template = get_document_type_classification_prompt()
+                prompt = template.format(
+                    doc_name=doc.get('name', 'Unknown'),
+                    content_preview=doc.get('content', '')[:500]  # First 500 chars for classification
+                )
+                messages = [HumanMessage(content=prompt)]
+                batch_inputs.append((messages, doc))
+
+            # Process batch using LangChain's built-in mechanisms
+            try:
+                logger.info(f"Processing classification batch {batch_num}/{total_batches} with {len(uncached_batch)} new documents")
+                batch_results = batch_processor.invoke(batch_inputs)
+
+                # Process results with individual document error handling
+                for idx, result in enumerate(batch_results):
+                    doc = result['item_info'].copy()
+
+                    if result['success'] and result['response']:
+                        # Successfully classified document type
+                        doc_type = result['response'].content.strip().lower()
+                        # Remove any "the document type is" prefix if present (for backward compatibility)
+                        if doc_type.startswith("the document type is "):
+                            doc_type = doc_type[21:].strip()
+                        doc['document_type'] = doc_type
+                        logger.debug(f"Classified '{doc.get('name', 'Unknown')}' as: {doc_type}")
+
+                        # Cache the result
+                        cache_key = f"classification:{doc.get('path', '')}"
+                        perf_manager.doc_cache.set(cache_key, doc, expire=86400 * 30)  # 30 days
+
+                        classified_docs.append(doc)
+                    else:
+                        # Fail on classification error
+                        error_msg = f"Failed to classify document '{doc.get('name', 'Unknown')}': {result.get('error', 'Unknown error')}"
+                        logger.error(error_msg)
+                        raise Exception(error_msg)
+
+            except Exception as e:
+                error_msg = f"Classification batch {batch_num} processing completely failed: {e}"
+                logger.error(error_msg)
+                raise Exception(error_msg)
+
+        # Add cached results to the final list
+        classified_docs.extend(cached_batch)
+
+    successful_classifications = len([d for d in classified_docs if d.get('document_type') != 'unknown document'])
+    success_rate = (successful_classifications / total_docs) * 100 if total_docs > 0 else 0
+    logger.info(f"✅ Classified {successful_classifications}/{total_docs} documents ({success_rate:.1f}% success rate)")
+
+    return classified_docs

app/ai/processing_pipeline.py

@@ -0,0 +1,279 @@
+#!/usr/bin/env python3
+"""
+Processing Pipeline Module
+
+This module contains content processing pipeline and workflow functions,
+including agent node functions and batch processing utilities.
+"""
+
+# Standard library imports
+import logging
+from typing import List, Dict, Optional
+
+# Third-party imports
+import streamlit as st
+from langchain_core.messages import HumanMessage, AIMessage
+from langchain_core.output_parsers import PydanticOutputParser
+from pydantic import BaseModel, Field
+
+# Local imports
+from app.ai.agent_utils import AgentState, create_batch_processor
+from app.ai.prompts import (
+    get_checklist_parsing_prompt,
+    get_document_relevance_prompt,
+    get_question_answering_prompt,
+    get_findings_summary_prompt,
+    get_description_generation_prompt,
+    get_document_summarization_prompt
+)
+from app.core.config import get_config
+from app.core.constants import DEFAULT_BATCH_SIZE
+
+logger = logging.getLogger(__name__)
+
+
+# Pydantic models for structured output parsing
+class ChecklistItem(BaseModel):
+    """Individual checklist item"""
+    text: str = Field(description="The checklist item text")
+    original: str = Field(description="The original text before any cleanup")
+
+class ChecklistCategory(BaseModel):
+    """Checklist category with items"""
+    name: str = Field(description="Category name (e.g., 'Organizational and Corporate Documents')")
+    items: List[ChecklistItem] = Field(description="List of checklist items in this category")
+
+class StructuredChecklist(BaseModel):
+    """Complete checklist with all categories"""
+    categories: Dict[str, ChecklistCategory] = Field(
+        description="Dictionary of categories keyed by letter (A, B, C, etc.)"
+    )
+
+class Question(BaseModel):
+    """Individual question"""
+    category: str = Field(description="Question category")
+    question: str = Field(description="The question text")
+    id: str = Field(description="Unique question ID")
+
+class StructuredQuestions(BaseModel):
+    """List of structured questions"""
+    questions: List[Question] = Field(description="List of all questions")
+
+
+
+def route_task(state: AgentState) -> AgentState:
+    """Route to appropriate task based on current state"""
+    messages = state["messages"]
+    if not messages:
+        return state
+
+    last_message = messages[-1].content if messages else ""
+
+    # Determine next action based on message content
+    if "parse" in last_message.lower() and "checklist" in last_message.lower():
+        state["next_action"] = "parse_checklist"
+    elif "analyze" in last_message.lower() or "match" in last_message.lower():
+        state["next_action"] = "match_checklist"
+    elif "?" in last_message:
+        state["next_action"] = "answer_question"
+    else:
+        state["next_action"] = "summarize"
+
+    return state
+
+
+def parse_checklist_node(state: AgentState, llm: "ChatAnthropic") -> AgentState:
+    """Parse checklist using structured output - standardized with StructuredChecklist!"""
+    messages = state["messages"]
+    checklist_text = messages[-1].content if messages else ""
+
+    # Set up structured parser - using the same as parse_checklist function
+    parser = PydanticOutputParser(pydantic_object=StructuredChecklist)
+    prompt = get_checklist_parsing_prompt()
+
+    try:
+        # Format the prompt with the checklist text and format instructions
+        formatted_prompt = prompt.format_messages(
+            checklist_text=checklist_text,  # Don't truncate - let LLM handle full checklist
+            format_instructions=parser.get_format_instructions()
+        )
+        
+        # Get LLM response
+        llm_response = llm.invoke(formatted_prompt)
+        
+        # Parse the response using the Pydantic parser
+        result = parser.parse(llm_response.content)
+
+        # Convert Pydantic model to expected dictionary format (same as parse_checklist)
+        categories_dict = {}
+        for key, category in result.categories.items():
+            categories_dict[key] = {
+                'name': category.name,
+                'items': [
+                    {
+                        'text': item.text,
+                        'original': item.original
+                    }
+                    for item in category.items
+                ]
+            }
+
+        state["checklist"] = categories_dict
+        state["messages"].append(AIMessage(content=f"Parsed {len(categories_dict)} categories"))
+
+    except Exception as e:
+        state["messages"].append(AIMessage(content=f"Parsing failed: {str(e)}"))
+
+    return state
+
+
+def match_checklist_node(state: AgentState, llm: "ChatAnthropic") -> AgentState:
+    """Match documents to checklist items - keep it simple"""
+    checklist = state.get("checklist", {})
+    documents = state.get("documents", [])
+
+    if not checklist or not documents:
+        state["messages"].append(AIMessage(content="Need both checklist and documents to match"))
+        return state
+
+    # For each checklist item, find relevant documents
+    findings = {}
+    for cat_letter, category in checklist.items():
+        cat_findings = []
+        for item in category.get("items", []):
+            # Use Claude to assess relevance
+            document_names = [d.get('name', 'Unknown') for d in documents[:10]]
+            prompt = get_document_relevance_prompt(item['text'], document_names)
+
+            response = llm.invoke([HumanMessage(content=str(prompt))])
+            cat_findings.append({
+                "item": item['text'],
+                "relevant_docs": response.content
+            })
+
+        findings[category['name']] = cat_findings
+
+    state["findings"] = findings
+    state["messages"].append(AIMessage(content=f"Matched checklist to {len(documents)} documents"))
+
+    return state
+
+
+def answer_question_node(state: AgentState, llm: "ChatAnthropic") -> AgentState:
+    """Answer questions using document context"""
+    messages = state["messages"]
+    question = messages[-1].content if messages else ""
+    documents = state.get("documents", [])
+
+    # Create context from documents
+    context = "\n".join([f"- {d.get('name', 'Unknown')}: {d.get('text', '')[:200]}"
+                        for d in documents[:5]])
+
+    prompt = get_question_answering_prompt(question, context)
+    response = llm.invoke([HumanMessage(content=prompt)])
+    state["messages"].append(AIMessage(content=response.content))
+
+    return state
+
+
+def summarize_node(state: AgentState, llm: "ChatAnthropic") -> AgentState:
+    """Summarize findings"""
+    findings = state.get("findings", {})
+
+    if not findings:
+        state["messages"].append(AIMessage(content="No findings to summarize"))
+        return state
+
+    prompt = get_findings_summary_prompt(findings)
+    response = llm.invoke([HumanMessage(content=prompt)])
+    state["messages"].append(AIMessage(content=response.content))
+
+    return state
+
+
+def route_condition(state: AgentState) -> str:
+    """Conditional routing function based on next_action"""
+    next_action = state.get("next_action")
+    if next_action == "parse_checklist":
+        return "parse_checklist"
+    elif next_action == "match_checklist":
+        return "match_checklist"
+    elif next_action == "answer_question":
+        return "answer_question"
+    else:
+        return "summarize"
+
+
+
+
+def batch_summarize_documents(documents: List[Dict], llm: "ChatAnthropic", batch_size: Optional[int] = None) -> List[Dict]:
+    """
+    Summarize documents using LangChain's built-in retry mechanisms and proper error handling.
+    Uses RunnableLambda for better batch processing control with individual item error handling.
+    Returns documents with added 'summary' field.
+
+    Args:
+        documents: List of document dictionaries to summarize
+        llm: ChatAnthropic instance for generating summaries
+        batch_size: Number of documents to process in each batch (uses config default if None)
+
+    Returns:
+        List of documents with added summary field
+    """
+
+    config = get_config()
+    if batch_size is None:
+        batch_size = DEFAULT_BATCH_SIZE
+
+    # Create batch processor with retry and fallback mechanisms
+    batch_processor = create_batch_processor(llm, max_concurrency=3)
+
+    # Process documents in batches
+    summarized_docs = []
+    total_docs = len(documents)
+    total_batches = (total_docs + batch_size - 1) // batch_size
+
+    for batch_num, i in enumerate(range(0, total_docs, batch_size), 1):
+        batch = documents[i:i + batch_size]
+        batch_end = min(i + batch_size, total_docs)
+
+        # Update progress with batch info
+        if hasattr(st, 'progress') and 'summary_progress' in st.session_state:
+            progress = i / total_docs
+            st.session_state.summary_progress.progress(
+                progress,
+                text=f"📝 Processing batch {batch_num}/{total_batches} (docs {i+1}-{batch_end} of {total_docs})"
+            )
+
+        # Prepare batch inputs for the processor
+        batch_inputs = []
+        for doc in batch:
+            template = get_document_summarization_prompt(doc)
+            prompt = template.format()
+            messages = [HumanMessage(content=prompt)]
+            batch_inputs.append((messages, doc))
+
+        # Process batch using LangChain's built-in mechanisms
+        try:
+            batch_results = batch_processor.invoke(batch_inputs)
+
+            # Process results with individual document error handling
+            for result in batch_results:
+                doc = result['item_info'].copy()
+
+                if result['success'] and result['response']:
+                    # Successfully generated summary
+                    doc['summary'] = result['response'].content.strip()
+                    summarized_docs.append(doc)
+                else:
+                    # Fail on summary generation error
+                    error_msg = f"Failed to generate summary for document '{doc.get('name', 'Unknown')}': {result.get('error', 'Unknown error')}"
+                    logger.error(error_msg)
+                    raise Exception(error_msg)
+
+        except Exception as e:
+            error_msg = f"Batch {batch_num} processing completely failed: {e}"
+            logger.error(error_msg)
+            raise Exception(error_msg)
+
+    return summarized_docs

{src → app}/ai/prompts.py

@@ -6,46 +6,75 @@ This module contains all prompt templates used for AI interactions
 in the DD-Checklist application.
 """
 
+# Standard library imports
 import json
 from typing import Dict, List
-from langchain_core.prompts import PromptTemplate, ChatPromptTemplate
+
+# Third-party imports
 from langchain_core.messages import SystemMessage, HumanMessage
+from langchain_core.prompts import PromptTemplate, ChatPromptTemplate, HumanMessagePromptTemplate, SystemMessagePromptTemplate
+
+# Local imports
+from app.core.constants import QA_MAX_TOKENS
 
 
-def get_checklist_parsing_prompt(checklist_text: str) -> ChatPromptTemplate:
-    """Generate prompt for parsing due diligence checklists with structured output"""
+def get_checklist_parsing_prompt() -> ChatPromptTemplate:
+    """Generate prompt template for parsing due diligence checklists with structured output"""
     return ChatPromptTemplate.from_messages([
         SystemMessage(content="""
-Parse this due diligence checklist into structured format. Extract:
-- Categories (A., B., C., etc.) with their names
-- Numbered items within each category (1., 2., 3., etc.)
-- Total count of items
-
-Follow the exact format specified in the format instructions.
+You are a JSON parser. Your ONLY task is to convert the checklist into valid JSON format.
+
+CRITICAL PARSING RULES:
+- Return ONLY valid JSON - no explanations, no notes, no additional text
+- Do NOT add any conversational text before or after the JSON
+- Do NOT offer to continue or ask questions
+- Do NOT provide partial results or examples
+- Parse the COMPLETE document - every single category and item
+
+JSON Structure Required:
+- Top-level object with "categories" field
+- Categories keyed by letter (A, B, C, D, E, etc.)
+- Each category has "name" and "items" fields
+- Each item has "text" and "original" fields
+
+You must process the ENTIRE checklist. Do not stop after a few categories.
+
+Output format:
+{
+  "categories": {
+    "A": {
+      "name": "Category Name",
+      "items": [
+        {"text": "Item text", "original": "1. Item text"}
+      ]
+    }
+  }
+}
+
+Return ONLY the JSON. No other text.
 """),
-        HumanMessage(content="""Parse this checklist:
+        HumanMessagePromptTemplate.from_template("""Parse this complete checklist into the exact JSON format:
 
 {checklist_text}
 
+Required JSON schema:
 {format_instructions}
 
-Please provide the structured output:""")
+Return the complete JSON with all categories found in the checklist:""")
     ])
 
 
 def get_document_relevance_prompt(item_text: str, documents: List[str]) -> PromptTemplate:
-    """Generate prompt for assessing document relevance to checklist items with structured output"""
+    """Generate prompt for assessing document relevance to checklist items"""
     return PromptTemplate.from_template(
-        """Analyze which documents are relevant to the following checklist item:
+        """Analyze which documents are most relevant to the following checklist item.
 
 Checklist Item: {item_text}
 
 Available Documents:
 {documents}
 
-{format_instructions}
-
-Please provide your analysis in the specified format:"""
+Provide a brief analysis identifying the most relevant documents and explain why they are relevant to this checklist item. Be concise and specific."""
     )
 
 
@@ -57,7 +86,7 @@ def get_question_answering_prompt(question: str, context: str) -> ChatPromptTemp
     ])
 
 
-def get_findings_summary_prompt(findings: Dict, max_chars: int = 2000) -> PromptTemplate:
+def get_findings_summary_prompt(findings: Dict, max_chars: int = QA_MAX_TOKENS) -> PromptTemplate:
     """Generate prompt for summarizing due diligence findings"""
     findings_text = json.dumps(findings, indent=2)[:max_chars]
     return PromptTemplate.from_template(
@@ -81,6 +110,24 @@ def get_description_generation_prompt(category_name: str, item_text: str) -> Pro
     ).partial(category_name=category_name, item_text=item_text)
 
 
+def get_document_type_classification_prompt() -> PromptTemplate:
+    """Generate prompt for fast document type classification based on first chunk content"""
+    return PromptTemplate.from_template(
+        "Classify the document type using one short phrase. Use exact terminology.\n"
+        "Respond with ONLY the document type, no prefix or explanation.\n\n"
+        "Examples:\n"
+        "certificate of incorporation\n"
+        "corporate bylaws\n"
+        "amended and restated bylaws\n"
+        "board resolution\n"
+        "financial statement\n"
+        "employment agreement\n"
+        "software license agreement\n\n"
+        "Document: {doc_name}\n"
+        "Content: {content_preview}\n\n"
+        "Document type:"
+    )
+
 def get_document_summarization_prompt(doc: Dict) -> PromptTemplate:
     """Generate prompt for document type identification and summarization"""
     doc_name = doc.get('name', 'Unknown')

app/core/__init__.py

@@ -0,0 +1,61 @@
+"""
+Core Business Logic Layer
+
+This layer contains the core business logic and domain models.
+It should not depend on UI or external frameworks.
+"""
+
+# Configuration
+from .config import AppConfig, get_config
+
+# Exceptions
+from .exceptions import (
+    AppException,
+    DocumentProcessingError,
+    SearchError,
+    ConfigError,
+    FileOperationError,
+    AIError,
+    LLMConnectionError,
+    LLMAuthenticationError,
+    LLMTimeoutError,
+    LLMQuotaExceededError,
+    LLMInvalidResponseError,
+    create_processing_error,
+    create_config_error,
+    create_ai_error
+)
+
+# Core classes and functions
+from .document_processor import DocumentProcessor
+from .search import search_and_analyze, search_documents
+from .ranking import rerank_results
+from .parsers import parse_checklist, parse_questions
+from .utils import create_document_processor, format_document_title, count_documents_in_directory
+from .logging import logger
+from .constants import (
+    RELEVANCY_THRESHOLD,
+    SIMILARITY_THRESHOLD,
+    DEFAULT_BATCH_SIZE,
+    QA_MAX_TOKENS,
+    CHECKLIST_PARSING_MAX_TOKENS
+)
+
+__all__ = [
+    # Configuration
+    'AppConfig', 'get_config',
+
+    # Exceptions
+    'AppException', 'DocumentProcessingError', 'SearchError', 'ConfigError',
+    'FileOperationError', 'AIError', 'LLMConnectionError', 'LLMAuthenticationError',
+    'LLMTimeoutError', 'LLMQuotaExceededError', 'LLMInvalidResponseError',
+    'create_processing_error', 'create_config_error', 'create_ai_error',
+
+    # Core functionality
+    'DocumentProcessor', 'search_and_analyze', 'search_documents', 'rerank_results',
+    'parse_checklist', 'parse_questions', 'create_document_processor',
+    'format_document_title', 'count_documents_in_directory', 'logger',
+
+    # Constants
+    'RELEVANCY_THRESHOLD', 'SIMILARITY_THRESHOLD', 'DEFAULT_BATCH_SIZE', 'QA_MAX_TOKENS', 'CHECKLIST_PARSING_MAX_TOKENS'
+]

app/core/config.py

@@ -0,0 +1,202 @@
+from typing import Dict, Any, Optional
+from pathlib import Path
+import os
+from dotenv import load_dotenv
+from app.core.constants import (
+    CHUNK_SIZE, CHUNK_OVERLAP, SIMILARITY_THRESHOLD,
+    RELEVANCY_THRESHOLD, CLASSIFICATION_MAX_TOKENS, CHECKLIST_PARSING_MAX_TOKENS,
+    TEMPERATURE
+)
+
+load_dotenv()
+
+
+class AppConfig:
+    def __init__(self) -> None:
+        self._config: Dict[str, Any] = {}
+        self._load_config()
+
+    def _load_config(self) -> None:
+        self._config['ui'] = {
+            'page_title': "🤖 AI Due Diligence",
+            'page_icon': "🤖",
+            'layout': "wide",
+            'top_k_search_results': 10
+        }
+
+        self._config['model'] = {
+            'sentence_transformer_model': 'sentence-transformers/all-mpnet-base-v2',
+            'claude_model': os.getenv('CLAUDE_MODEL', 'claude-3-5-sonnet'),
+            'claude_haiku_model': 'claude-3-5-haiku-20241022',
+            'classification_max_tokens': CLASSIFICATION_MAX_TOKENS,
+            'temperature': float(os.getenv('CLAUDE_TEMPERATURE', str(TEMPERATURE))),
+            'max_tokens': int(os.getenv('CLAUDE_MAX_TOKENS', '16000'))  # High limit for checklist parsing
+        }
+
+        self._config['processing'] = {
+            'chunk_size': CHUNK_SIZE,
+            'chunk_overlap': CHUNK_OVERLAP,
+            'similarity_threshold': SIMILARITY_THRESHOLD,
+            'relevancy_threshold': RELEVANCY_THRESHOLD,
+            'supported_file_extensions': [
+                '.pdf', '.docx', '.doc', '.txt', '.md',
+                '.xls', '.xlsx', '.ppt', '.pptx'
+            ],
+            'faiss_store_name': 'default'
+        }
+
+        self._config['paths'] = {
+            'data_dir': Path('data'),
+            'strategy_dir': Path('data/strategy'),
+            'checklist_dir': Path('data/checklist'),
+            'questions_dir': Path('data/questions'),
+            'vdrs_dir': Path('data/vdrs'),
+            'faiss_dir': Path('data/search_indexes')
+        }
+
+        self._config['anthropic'] = {
+            'api_key': os.getenv('ANTHROPIC_API_KEY'),
+            'model': os.getenv('CLAUDE_MODEL', 'claude-3-5-sonnet')
+        }
+
+    @property
+    def ui(self) -> Dict[str, Any]:
+        return self._config['ui']
+
+    @property
+    def model(self) -> Dict[str, Any]:
+        return self._config['model']
+
+    @property
+    def processing(self) -> Dict[str, Any]:
+        return self._config['processing']
+
+    @property
+    def paths(self) -> Dict[str, Path]:
+        return self._config['paths']
+
+    @property
+    def anthropic(self) -> Dict[str, Optional[str]]:
+        return self._config['anthropic']
+
+    def validate(self) -> bool:
+        """Validate all critical configuration values."""
+        self._validate_anthropic_config()
+        self._validate_paths()
+        self._validate_models()
+        self._validate_processing_config()
+        self._validate_file_extensions()
+        return True
+
+    def _validate_anthropic_config(self) -> None:
+        """Validate Anthropic API configuration."""
+        if not self.anthropic.get('api_key'):
+            raise ValueError("ANTHROPIC_API_KEY environment variable is required")
+
+        model = self.anthropic.get('model')
+        if not model:
+            raise ValueError("CLAUDE_MODEL environment variable is required")
+
+        valid_claude_models = [
+            'claude-3-5-sonnet',
+            'claude-3-5-haiku-20241022',
+            'claude-3-opus-20240229',
+            'claude-3-sonnet-20240229',
+            'claude-3-haiku-20240307'
+        ]
+        if model not in valid_claude_models:
+            raise ValueError(f"Invalid Claude model: {model}. Valid models: {', '.join(valid_claude_models)}")
+
+    def _validate_paths(self) -> None:
+        """Validate that critical directories exist."""
+        critical_dirs = [
+            ('data_dir', self.paths['data_dir']),
+            ('vdrs_dir', self.paths['vdrs_dir'])
+        ]
+
+        for dir_name, dir_path in critical_dirs:
+            if not dir_path.exists():
+                raise ValueError(f"Critical directory '{dir_name}' does not exist: {dir_path}")
+            if not dir_path.is_dir():
+                raise ValueError(f"Path '{dir_name}' exists but is not a directory: {dir_path}")
+
+    def _validate_models(self) -> None:
+        """Validate that required models are available."""
+        # Check sentence transformer model
+        model_path = Path('models') / 'sentence_transformers' / self.model['sentence_transformer_model'].split('/')[-1]
+        if not model_path.exists():
+            raise ValueError(f"Sentence transformer model not found: {model_path}")
+
+        # Check cross-encoder model
+        cross_encoder_path = Path('models') / 'cross_encoder' / 'ms-marco-MiniLM-L-6-v2'
+        if not cross_encoder_path.exists():
+            raise ValueError(f"Cross-encoder model not found: {cross_encoder_path}")
+
+    def _validate_processing_config(self) -> None:
+        """Validate processing configuration values."""
+        processing = self.processing
+
+        # Validate chunk size
+        chunk_size = processing['chunk_size']
+        if not isinstance(chunk_size, int) or chunk_size <= 0:
+            raise ValueError(f"Invalid chunk_size: {chunk_size}. Must be a positive integer.")
+
+        # Validate chunk overlap
+        chunk_overlap = processing['chunk_overlap']
+        if not isinstance(chunk_overlap, int) or chunk_overlap < 0:
+            raise ValueError(f"Invalid chunk_overlap: {chunk_overlap}. Must be a non-negative integer.")
+        if chunk_overlap >= chunk_size:
+            raise ValueError(f"chunk_overlap ({chunk_overlap}) must be less than chunk_size ({chunk_size})")
+
+        # Validate thresholds
+        similarity_threshold = processing['similarity_threshold']
+        if not isinstance(similarity_threshold, (int, float)) or not (0 <= similarity_threshold <= 1):
+            raise ValueError(f"Invalid similarity_threshold: {similarity_threshold}. Must be between 0 and 1.")
+
+        relevancy_threshold = processing['relevancy_threshold']
+        if not isinstance(relevancy_threshold, (int, float)) or not (0 <= relevancy_threshold <= 1):
+            raise ValueError(f"Invalid relevancy_threshold: {relevancy_threshold}. Must be between 0 and 1.")
+
+        # Validate max tokens
+        max_tokens = processing.get('classification_max_tokens', CLASSIFICATION_MAX_TOKENS)
+        if not isinstance(max_tokens, int) or max_tokens <= 0:
+            raise ValueError(f"Invalid classification_max_tokens: {max_tokens}. Must be a positive integer.")
+
+    def _validate_file_extensions(self) -> None:
+        """Validate supported file extensions."""
+        extensions = self.processing['supported_file_extensions']
+        if not extensions:
+            raise ValueError("supported_file_extensions cannot be empty")
+
+        # Validate each extension starts with a dot and contains valid characters
+        for ext in extensions:
+            if not isinstance(ext, str):
+                raise ValueError(f"Invalid file extension type: {type(ext)}. Must be string.")
+            if not ext.startswith('.'):
+                raise ValueError(f"File extension must start with '.': {ext}")
+            if len(ext) < 2 or not ext[1:].replace('_', '').replace('-', '').isalnum():
+                raise ValueError(f"Invalid file extension format: {ext}")
+
+    def get_supported_extensions(self) -> list[str]:
+        """Get list of supported file extensions for document processing."""
+        return self._config['processing']['supported_file_extensions']
+
+
+# Global configuration instance
+_config_instance: Optional[AppConfig] = None
+
+
+def get_app_config() -> AppConfig:
+    """Get the global application configuration instance."""
+    global _config_instance
+    if _config_instance is None:
+        _config_instance = AppConfig()
+        _config_instance.validate()
+    return _config_instance
+
+
+# Compatibility alias
+init_app_config = get_app_config
+
+# Compatibility alias
+get_config = get_app_config

app/core/constants.py

@@ -0,0 +1,24 @@
+# Constants for the application
+
+# Chunk sizes
+CHUNK_SIZE = 1000
+CHUNK_OVERLAP = 200
+
+# Thresholds
+SIMILARITY_THRESHOLD = 0.2
+RELEVANCY_THRESHOLD = 0.25
+
+# Token limits
+CLASSIFICATION_MAX_TOKENS = 1000
+QA_MAX_TOKENS = 8000
+CHECKLIST_PARSING_MAX_TOKENS = 16000  # Large enough for full checklist parsing
+
+# AI Model Configuration
+TEMPERATURE = 0.0  # Deterministic responses for due diligence consistency
+
+# Batch sizes
+DEFAULT_BATCH_SIZE = 10
+CLASSIFICATION_BATCH_SIZE = 20
+
+# AI Analysis types
+SUPPORTED_ANALYSIS_TYPES = ["overview", "strategic", "checklist", "questions"]

app/core/content_ingestion.py

@@ -0,0 +1,282 @@
+#!/usr/bin/env python3
+"""
+Unified Content Ingestion System
+
+This module provides a unified processing pipeline with simple ingestion functions.
+All content types (VDR documents, markdown files, etc.) go through the same processing pipeline
+with different ingestion functions handling the content-specific parsing.
+"""
+
+# Standard library imports
+import json
+import logging
+import time
+from pathlib import Path
+from typing import List, Dict, Any, Optional, Tuple, Callable
+
+# Third-party imports
+from langchain_core.documents import Document
+from langchain_community.vectorstores import FAISS
+from tqdm import tqdm
+
+# Local imports
+from app.core.config import get_config
+from app.core.model_cache import get_cached_embeddings
+from app.core.parsers import parse_checklist, parse_questions
+
+logger = logging.getLogger(__name__)
+
+
+def vdr_ingest(vdr_path: Path, store_name: str, llm=None) -> Tuple[List[Document], Dict[str, Any]]:
+    """Ingest VDR documents using DocumentProcessor"""
+    logger.info(f"Ingesting VDR documents from {vdr_path}")
+
+    # Count total files for progress tracking
+    total_files = sum(1 for f in vdr_path.rglob('*')
+                     if f.is_file() and f.suffix.lower() in ['.pdf', '.docx', '.doc', '.txt', '.md'])
+
+    # Initialize document processor
+    from app.core.utils import create_document_processor
+    processor = create_document_processor(store_name=store_name)
+
+    # Process the data room with file-level progress
+    with tqdm(total=total_files, desc=f"Files in {store_name}",
+              unit="files", leave=False) as file_pbar:
+
+        result = processor.load_data_room(str(vdr_path))
+
+        # Update progress bar based on actual files processed
+        if file_pbar and result.get('documents_count', 0) > 0:
+            file_pbar.update(result['documents_count'])
+
+    metadata = {
+        'content_type': 'vdr',
+        'source_path': str(vdr_path),
+        'total_files': total_files,
+        **result
+    }
+
+    return processor.documents, metadata
+
+
+def classify_vdr_documents(documents: List[Document], store_name: str, classifier=None) -> Dict[str, str]:
+    """Classify VDR documents using fast Haiku classifier"""
+    if not classifier or not documents:
+        return {}
+
+    logger.info(f"🏷️ Classifying document types for {store_name}")
+
+    # Extract only first chunks for classification efficiency
+    first_chunks = []
+    for doc in documents:
+        if doc.metadata.get('is_first_chunk', False):
+            first_chunks.append({
+                'name': doc.metadata.get('name', ''),
+                'path': doc.metadata.get('path', ''),
+                'content': doc.page_content[:800]
+            })
+
+    if not first_chunks:
+        logger.warning(f"⚠️ No first chunks found for classification in {store_name}")
+        return {}
+
+    try:
+        from app.ai.document_classifier import batch_classify_document_types
+        classified_docs = batch_classify_document_types(first_chunks, classifier)
+
+        # Build classifications dictionary
+        classifications = {}
+        for doc in classified_docs:
+            if 'document_type' in doc and doc['path']:
+                classifications[doc['path']] = doc['document_type']
+
+        logger.info(f"✅ Classified {len(classifications)} document types for {store_name}")
+        return classifications
+
+    except Exception as e:
+        logger.error(f"⚠️ Failed to classify document types for {store_name}: {e}")
+        return {}
+
+
+def process_content(content_source: Any, content_type: str, store_name: str, classifier=None, llm=None) -> Dict[str, Any]:
+    """Process content source into FAISS index"""
+    start_time = time.time()
+
+    try:
+        # Get ingestion function
+        ingest_func = get_ingestion_function(content_type)
+        documents, ingestion_metadata = ingest_func(content_source, store_name, llm)
+
+        if not documents:
+            return {
+                'success': False,
+                'store_name': store_name,
+                'error': 'No documents extracted'
+            }
+
+        # Classify VDR documents if classifier provided
+        classifications = {}
+        if classifier and content_type == 'vdr':
+            classifications = classify_vdr_documents(documents, store_name, classifier)
+
+        # Create FAISS index
+        from app.core.model_cache import get_cached_embeddings
+        from app.core.config import get_config
+        config = get_config()
+        embeddings = get_cached_embeddings(config.model['sentence_transformer_model'])
+        vector_store = FAISS.from_documents(documents, embeddings)
+
+        # Save index
+        faiss_dir = config.paths['faiss_dir']
+        faiss_dir.mkdir(parents=True, exist_ok=True)
+        vector_store.save_local(str(faiss_dir), index_name=store_name)
+
+        # Save classifications if available
+        if classifications:
+            classifications_file = faiss_dir / f"{store_name}_document_types.json"
+            classifications_file.write_text(
+                json.dumps(classifications, indent=2, ensure_ascii=False)
+            )
+
+        # Save enhanced checklists
+        if 'enhanced_checklists' in ingestion_metadata:
+            checklists_file = faiss_dir / "checklists.json"
+            checklists_file.write_text(
+                json.dumps(ingestion_metadata['enhanced_checklists'], indent=2, ensure_ascii=False)
+            )
+
+        processing_time = time.time() - start_time
+
+        return {
+            'success': True,
+            'store_name': store_name,
+            'processing_time': processing_time,
+            'classifications_count': len(classifications),
+            **ingestion_metadata
+        }
+
+    except Exception as e:
+        return {
+            'success': False,
+            'store_name': store_name,
+            'error': str(e),
+            'processing_time': time.time() - start_time
+        }
+
+
+def checklist_ingest(content_dir: Path, store_name: str, llm=None) -> Tuple[List[Document], Dict[str, Any]]:
+    """Ingest checklist markdown files"""
+    logger.info(f"Ingesting checklist files from {content_dir}")
+
+    if not content_dir.exists():
+        raise FileNotFoundError(f"Checklist directory not found: {content_dir}")
+
+    # Find all markdown files
+    md_files = list(content_dir.glob("*.md"))
+    if not md_files:
+        raise ValueError(f"No markdown files found in {content_dir}")
+
+    all_documents = []
+
+    with tqdm(md_files, desc="Processing checklist files",
+              unit="file", leave=False) as file_pbar:
+
+        for md_file in file_pbar:
+            file_pbar.set_description(f"Processing {md_file.name}")
+            logger.info(f"Processing: {md_file.name}")
+
+            content = md_file.read_text(encoding='utf-8')
+            parsed_data = parse_checklist(content, llm)
+
+            # Convert checklist items to documents
+            for cat_key, category in parsed_data.items():
+                for item in category.get('items', []):
+                    doc = Document(
+                        page_content=item['text'],
+                        metadata={
+                            'source': md_file.name,
+                            'category': category['name'],
+                            'type': 'checklist_item'
+                        }
+                    )
+                    all_documents.append(doc)
+
+    metadata = {
+        'content_type': 'checklist',
+        'source_path': str(content_dir),
+        'md_files_count': len(md_files),
+        'documents_count': len(all_documents)
+    }
+
+    return all_documents, metadata
+
+
+def questions_ingest(content_dir: Path, store_name: str, llm=None) -> Tuple[List[Document], Dict[str, Any]]:
+    """Ingest questions markdown files"""
+    logger.info(f"Ingesting questions files from {content_dir}")
+
+    if not content_dir.exists():
+        raise FileNotFoundError(f"Questions directory not found: {content_dir}")
+
+    # Find all markdown files
+    md_files = list(content_dir.glob("*.md"))
+    if not md_files:
+        raise ValueError(f"No markdown files found in {content_dir}")
+
+    all_documents = []
+
+    with tqdm(md_files, desc="Processing questions files",
+              unit="file", leave=False) as file_pbar:
+
+        for md_file in file_pbar:
+            file_pbar.set_description(f"Processing {md_file.name}")
+            logger.info(f"Processing: {md_file.name}")
+
+            content = md_file.read_text(encoding='utf-8')
+            parsed_data = parse_questions(content, llm)
+
+            # Convert questions to documents
+            for question in parsed_data:
+                doc = Document(
+                    page_content=f"{question['category']}: {question['question']}",
+                    metadata={
+                        'source': md_file.name,
+                        'category': question['category'],
+                        'question_id': question['id'],
+                        'type': 'question'
+                    }
+                )
+                all_documents.append(doc)
+
+    metadata = {
+        'content_type': 'questions',
+        'source_path': str(content_dir),
+        'md_files_count': len(md_files),
+        'documents_count': len(all_documents)
+    }
+
+    return all_documents, metadata
+
+
+# Factory function for getting ingestion functions
+def get_ingestion_function(content_type: str) -> Callable[..., Tuple[List[Document], Dict[str, Any]]]:
+    """Factory function to get appropriate ingestion function"""
+    functions = {
+        'vdr': vdr_ingest,
+        'checklist': checklist_ingest,
+        'questions': questions_ingest
+    }
+
+    if content_type not in functions:
+        raise ValueError(f"Unknown content type: {content_type}. Available: {list(functions.keys())}")
+
+    return functions[content_type]
+
+
+# Backward compatibility - create UnifiedContentProcessor class that uses process_content
+class UnifiedContentProcessor:
+    """Backward compatibility wrapper for process_content function"""
+
+    def process_content_source(self, content_source: Any, content_type: str, store_name: str, classifier=None, progress_bar=None, llm=None):
+        """Process content using the unified function"""
+        return process_content(content_source, content_type, store_name, classifier, llm)

src/document_processing.py → app/core/document_processor.py

@@ -2,19 +2,18 @@
 """
 Streamlined Document Processing Module
 
-This module provides a simplified document processing pipeline with:
-- Direct LangChain loader integration with glob patterns  
+This module provides a document processing pipeline with:
+- Direct LangChain loader integration with glob patterns
 - Built-in FAISS vector storage without external file tracking
 - Semantic text chunking using RecursiveCharacterTextSplitter
 - Consolidated document metadata handling
 """
 
 import os
-import logging
+import time
 
-# Fix tokenizers parallelism warning
-os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
-import re
+# Enable tokenizers parallelism for better performance
+os.environ.setdefault("TOKENIZERS_PARALLELISM", "true")
 
 from pathlib import Path
 from typing import Dict, List, Optional, Any, Callable
@@ -23,17 +22,23 @@ from datetime import datetime
 # LangChain imports
 from langchain_community.document_loaders import DirectoryLoader, PyPDFLoader, Docx2txtLoader, TextLoader
 from langchain_community.vectorstores import FAISS
-from langchain_huggingface import HuggingFaceEmbeddings
 from langchain_core.documents import Document
 from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_huggingface import HuggingFaceEmbeddings
 
-# Import configuration
-from .config import get_config
-
-# Import error handling  
-
+# Import configuration and utilities from app modules
+from app.core.config import get_app_config
+from app.core.model_cache import get_cached_embeddings
+from app.core.logging import logger
+from app.core.performance import get_performance_manager, monitor_performance, cached_by_content
 
-logger = logging.getLogger(__name__)
+# Optional accelerate import
+try:
+    from accelerate import Accelerator
+    ACCELERATE_AVAILABLE = True
+except ImportError:
+    ACCELERATE_AVAILABLE = False
+    Accelerator = None
 
 
 # =============================================================================
@@ -43,13 +48,13 @@ logger = logging.getLogger(__name__)
 def safe_execute(func: Callable, default: Any = None, context: str = "", log_errors: bool = True) -> Any:
     """
     Execute a function with basic error handling and logging
-    
+
     Args:
         func: Function to execute
         default: Value to return on error
         context: Brief description for logs
         log_errors: Whether to log errors
-        
+
     Returns:
         Function result or default value on error
     """
@@ -78,71 +83,98 @@ def escape_markdown_math(text: str) -> str:
 class DocumentProcessor:
     """
     Streamlined document processing class with integrated FAISS vector storage
-    
+
     This class consolidates all document processing functionality including:
     - Document loading using LangChain's DirectoryLoader with glob patterns
     - Semantic text chunking with RecursiveCharacterTextSplitter
     - FAISS vector storage for similarity search
     - Document metadata handling
     """
-    
+
     def __init__(self, model_name: Optional[str] = None, store_name: Optional[str] = None):
         """
         Initialize the document processor
-        
+
         Args:
             model_name: Name of the sentence transformer model for embeddings (optional)
             store_name: Name for the FAISS store (optional, uses config default)
         """
-        config = get_config()
-        self.model_name = model_name or config.model.sentence_transformer_model
-        self.store_name = store_name or config.processing.faiss_store_name
-        
+        config = get_app_config()
+        self.model_name = model_name or config.model['sentence_transformer_model']
+        self.store_name = store_name or config.processing['faiss_store_name']
+
         # Initialize components
         self.documents: List[Document] = []
         self.vector_store: Optional[FAISS] = None
         self.embeddings: Optional[HuggingFaceEmbeddings] = None
         self.text_splitter: Optional[RecursiveCharacterTextSplitter] = None
         self.performance_stats = {}
-        
+
         # Convenience properties for backward compatibility
         self.chunks = []  # Will be populated after processing
-        
+
         # Initialize text splitter with semantic boundaries
         self._init_text_splitter()
-        
+
         # Initialize embeddings if model name provided
         if self.model_name:
-            self.embeddings = HuggingFaceEmbeddings(model_name=self.model_name)
-            logger.info(f"Initialized embeddings with model: {self.model_name}")
+            self.embeddings = get_cached_embeddings(self.model_name)
+            logger.info(f"Initialized cached embeddings with model: {self.model_name}")
+
+            # Setup accelerate for GPU optimization if available
+            if ACCELERATE_AVAILABLE:
+                try:
+                    self.accelerator = Accelerator()
+                    logger.info(f"Accelerate initialized with device: {self.accelerator.device}")
+                except Exception as e:
+                    logger.warning(f"Failed to initialize accelerate: {e}")
+                    self.accelerator = None
+            else:
+                self.accelerator = None
         else:
             logger.warning("No model name provided - embeddings not initialized")
-        
+            self.accelerator = None
+
         # Try to load existing FAISS store
         self._load_existing_store()
-    
+
     def _init_text_splitter(self):
         """Initialize the text splitter with optimal settings for semantic chunking"""
-        config = get_config()
+        config = get_app_config()
         self.text_splitter = RecursiveCharacterTextSplitter(
-            chunk_size=config.processing.chunk_size,
-            chunk_overlap=config.processing.chunk_overlap,
-            separators=["\\n\\n", "\\n", ".", "!", "?", ",", " "],
+            chunk_size=config.processing['chunk_size'],
+            chunk_overlap=config.processing['chunk_overlap'],
+            # Better separators for business documents with semantic boundaries
+            separators=[
+                "\n\n\n",  # Triple newlines (major section breaks)
+                "\n\n",    # Double newlines (paragraph breaks)
+                "\n",      # Single newlines
+                ". ",      # Sentences
+                ".\n",     # Sentences with newlines
+                "! ",      # Exclamations
+                "? ",      # Questions
+                "; ",      # Semicolons (common in legal/business docs)
+                ", ",      # Commas (last resort for long sentences)
+                " ",       # Spaces
+                "",        # Character level (absolute last resort)
+            ],
             length_function=len,
             is_separator_regex=False,
+            # Keep related content together
+            keep_separator=True,  # Keep separators to maintain context
         )
-        logger.info(f"Initialized text splitter: {config.processing.chunk_size} chars, {config.processing.chunk_overlap} overlap")
-    
+        logger.info(f"Initialized semantic text splitter: {config.processing['chunk_size']} chars, {config.processing['chunk_overlap']} overlap")
+
     def _load_existing_store(self):
         """Load existing FAISS store if available"""
         if not self.embeddings:
             return
-        
-        config = get_config()
-        faiss_dir = Path(config.paths.data_dir) / "enhanced_faiss"
+
+        config = get_app_config()
+        faiss_dir = config.paths['faiss_dir']
         faiss_index_path = faiss_dir / f"{self.store_name}.faiss"
         faiss_pkl_path = faiss_dir / f"{self.store_name}.pkl"
-        
+
         try:
             if faiss_index_path.exists() and faiss_pkl_path.exists():
                 self.vector_store = FAISS.load_local(
@@ -157,60 +189,54 @@ class DocumentProcessor:
         except Exception as e:
             logger.error(f"Failed to load FAISS store: {e}")
             self.vector_store = None
-    
-    def _save_store(self):
-        """Save FAISS store to disk"""
-        if not self.vector_store:
-            return
-        
-        try:
-            config = get_config()
-            faiss_dir = Path(config.paths.data_dir) / "enhanced_faiss"
-            faiss_dir.mkdir(parents=True, exist_ok=True)
-            
-            self.vector_store.save_local(
-                str(faiss_dir),
-                index_name=self.store_name
-            )
-            logger.info(f"Saved FAISS store: {self.store_name} with {self.vector_store.index.ntotal} vectors")
-        except Exception as e:
-            logger.error(f"Failed to save FAISS store: {e}")
-    
+
+    @monitor_performance
     def load_data_room(self, data_room_path: str, progress_bar=None) -> Dict[str, Any]:
         """
         Load and process an entire data room using DirectoryLoader with glob patterns
-        
+
         Args:
             data_room_path: Path to the data room directory
             progress_bar: Optional Streamlit progress bar object
-            
+
         Returns:
             Dictionary with processing results including performance metrics
         """
         import time
         start_time = time.time()
-        
-        config = get_config()
+
+        config = get_app_config()
         data_room_path = Path(data_room_path)
-        
+
         if not data_room_path.exists():
             logger.error(f"Data room path does not exist: {data_room_path}")
             return {'documents_count': 0, 'chunks_count': 0, 'has_embeddings': False}
-        
+
         logger.info(f"Starting streamlined data room processing: {data_room_path}")
-        
+
         # Clear existing documents
         self.documents = []
+
+    @monitor_performance
+    def load_data_room(self, data_room_path: str, progress_bar=None) -> Dict[str, Any]:
+        start_time = time.time()
         documents_loaded = 0
-        
+        config = get_app_config()
+
         # Load documents by file type using DirectoryLoader with glob patterns
-        supported_extensions = config.processing.supported_file_extensions
-        
+        supported_extensions = config.processing['supported_file_extensions']
+        perf_manager = get_performance_manager()
+
+        # Get memory info for batch optimization
+        mem_info = perf_manager.monitor_memory_usage()
+        logger.info(f"Memory usage at start: {mem_info['percent']:.1f}%")
+        logger.info(f"Available memory: {mem_info['rss']:.1f}MB")
+
         for ext in supported_extensions:
             try:
                 # Create glob pattern for this extension
                 glob_pattern = f"**/*{ext}"
-                
+
                 # Choose appropriate loader based on extension
                 if ext == '.pdf':
                     loader_cls = PyPDFLoader
@@ -220,7 +246,7 @@ class DocumentProcessor:
                     loader_cls = TextLoader
                 else:
                     continue
-                
+
                 # Use DirectoryLoader with glob pattern
                 loader = DirectoryLoader(
                     str(data_room_path),
@@ -231,14 +257,14 @@ class DocumentProcessor:
                     show_progress=False,  # Disable verbose progress output
                     use_multithreading=True
                 )
-                
+
                 # Load documents for this extension
                 docs = safe_execute(
                     lambda: loader.load(),
                     default=[],
                     context=f"Loading {ext} files"
                 )
-                
+
                 if docs:
                     # Add relative path information to metadata
                     for doc in docs:
@@ -253,34 +279,55 @@ class DocumentProcessor:
                                     # If relative path fails, use original source
                                     doc.metadata['path'] = doc.metadata['source']
                                     doc.metadata['name'] = source_path.name
-                    
+
                     self.documents.extend(docs)
                     documents_loaded += len(docs)
                     logger.info(f"Loaded {len(docs)} {ext} documents")
-                
+
+                    # Monitor memory usage and trigger GC if needed
+                    mem_usage = perf_manager.monitor_memory_usage()
+                    if perf_manager.should_gc_collect(mem_usage):
+                        import gc
+                        gc.collect()
+                        logger.debug(f"GC triggered - memory usage: {mem_usage['rss']:.1f}MB")
             except Exception as e:
                 logger.error(f"Error loading {ext} files: {e}")
-        
+
         scan_time = time.time() - start_time
         logger.info(f"Document loading completed in {scan_time:.2f} seconds")
-        
+
         # Split documents into chunks using the text splitter
         chunk_start = time.time()
         if self.documents and self.text_splitter:
+            # Track original documents to identify first chunks
+            original_docs = {doc.metadata.get('source', ''): True for doc in self.documents}
+
             self.documents = self.text_splitter.split_documents(self.documents)
-            
+
             # Add chunk metadata and populate chunks for backward compatibility
+            # Track which documents we've seen to mark first chunks
+            seen_documents = {}
             self.chunks = []
+
             for i, doc in enumerate(self.documents):
                 doc.metadata['chunk_id'] = f"chunk_{i}"
                 doc.metadata['processed_at'] = datetime.now().isoformat()
-                
+
+                # Mark first chunks for each document (critical for document type matching)
+                doc_source = doc.metadata.get('source', '')
+                if doc_source not in seen_documents:
+                    doc.metadata['is_first_chunk'] = True
+                    seen_documents[doc_source] = True
+                    logger.debug(f"First chunk marked for: {doc_source}")
+                else:
+                    doc.metadata['is_first_chunk'] = False
+
                 # Add citation information if available
                 if 'page' in doc.metadata:
                     doc.metadata['citation'] = f"page {doc.metadata['page']}"
                 else:
                     doc.metadata['citation'] = doc.metadata.get('name', 'document')
-                
+
                 # Create chunk dict for backward compatibility
                 chunk_dict = {
                     'text': doc.page_content,
@@ -290,33 +337,29 @@ class DocumentProcessor:
                     'metadata': doc.metadata
                 }
                 self.chunks.append(chunk_dict)
-        
+
+            first_chunks_count = len([doc for doc in self.documents if doc.metadata.get('is_first_chunk', False)])
+            logger.info(f"Marked {first_chunks_count} first chunks out of {len(self.documents)} total chunks")
+
         chunk_time = time.time() - chunk_start
         logger.info(f"Text splitting completed in {chunk_time:.2f} seconds")
-        
-        # Create or update FAISS vector store
+
+                # FAISS vector store should be loaded from pre-built indices
         embedding_time = 0
         if self.embeddings and self.documents:
             embedding_start = time.time()
-            
+
             if self.vector_store is None:
-                # Create new FAISS store
-                self.vector_store = FAISS.from_documents(self.documents, self.embeddings)
-                logger.info(f"Created new FAISS store with {len(self.documents)} documents")
+                logger.debug("FAISS store not pre-loaded (expected during index building)")
             else:
-                # Add documents to existing store
-                self.vector_store.add_documents(self.documents)
-                logger.info(f"Added {len(self.documents)} documents to existing FAISS store")
-            
-            # Save the updated store
-            self._save_store()
-            
+                logger.info(f"Using pre-loaded FAISS store with {self.vector_store.index.ntotal} vectors")
+
             embedding_time = time.time() - embedding_start
-            logger.info(f"FAISS processing completed in {embedding_time:.2f} seconds")
-        
+            logger.info(f"FAISS check completed in {embedding_time:.2f} seconds")
+
         total_time = time.time() - start_time
         logger.info(f"Total data room processing completed in {total_time:.2f} seconds")
-        
+
         # Store performance stats
         self.performance_stats = {
             'total_time': total_time,
@@ -325,7 +368,7 @@ class DocumentProcessor:
             'embedding_time': embedding_time,
             'documents_per_second': documents_loaded / scan_time if scan_time > 0 else 0
         }
-        
+
         return {
             'documents_count': documents_loaded,
             'chunks_count': len(self.documents),
@@ -333,65 +376,80 @@ class DocumentProcessor:
             'has_embeddings': self.vector_store is not None,
             'performance': self.performance_stats
         }
-    
+
     def search(self, query: str, top_k: int = 5, threshold: Optional[float] = None) -> List[Dict]:
         """
         Search documents using FAISS similarity search
-        
+
         Args:
             query: Search query
             top_k: Number of top results to return
             threshold: Minimum similarity threshold
-            
+
         Returns:
             List of search results with scores and metadata
         """
         if not self.vector_store:
             logger.warning("FAISS vector store not available for search")
             return []
-        
-        config = get_config()
+
+        config = get_app_config()
         if threshold is None:
-            threshold = config.processing.similarity_threshold
-        
+            threshold = config.processing['similarity_threshold']
+
         try:
-            # Perform similarity search with scores
-            docs_and_scores = self.vector_store.similarity_search_with_score(query, k=top_k*2)
-            
-            results = []
+            # Perform similarity search with scores - get more candidates for reranking
+            docs_and_scores = self.vector_store.similarity_search_with_score(query, k=max(20, top_k*3))
+
+            # Initial filtering and conversion to candidates format
+            candidates = []
             seen_texts = set()
-            
+
             for doc, score in docs_and_scores:
                 # Convert FAISS distance to similarity score (higher is better)
-                similarity_score = 1.0 / (1.0 + score) if score >= 0 else 1.0
-                
+                similarity_score = 1.0 - (score / 2.0) if score <= 2.0 else 0.0
+
                 if similarity_score < threshold:
                     continue
-                
+
                 # Avoid duplicates based on text content
                 text_preview = doc.page_content[:100]
                 if text_preview not in seen_texts:
                     seen_texts.add(text_preview)
-                    
-                    results.append({
+
+                    candidates.append({
                         'text': doc.page_content,
                         'source': doc.metadata.get('name', ''),
                         'path': doc.metadata.get('path', ''),
-                        'full_path': doc.metadata.get('source', ''),
-                        'citation': doc.metadata.get('citation', 'document'),
                         'score': float(similarity_score),
                         'metadata': doc.metadata
                     })
-                    
-                    if len(results) >= top_k:
-                        break
-            
+
+            # Apply reranking if we have candidates
+            if candidates:
+                try:
+                    # Import rerank_results from ranking module to avoid circular import
+                    from app.core.ranking import rerank_results
+
+                    # Rerank the top candidates (limit to reasonable number for performance)
+                    candidates_to_rerank = candidates[:min(15, len(candidates))]  # Rerank up to 15 candidates
+
+                    reranked_results = rerank_results(query, candidates_to_rerank)
+                    results = reranked_results[:top_k]  # Take top_k after reranking
+                    logger.info(f"Reranked {len(reranked_results)} search results for query: {query[:50]}...")
+                except Exception as e:
+                    # Reranking failed - use original results without reranking
+                    logger.warning(f"Reranking failed for search query '{query}': {e}. Using original similarity scores.")
+                    results = candidates[:top_k]
+            else:
+                results = []
+
             return results
-            
+
         except Exception as e:
             logger.error(f"Failed to search FAISS store: {e}")
-            return []
-    
+            raise RuntimeError(f"Document search failed for query '{query}': {e}") from e
+
     def get_statistics(self) -> Dict[str, Any]:
         """Get processing statistics"""
         stats = {
@@ -401,10 +459,9 @@ class DocumentProcessor:
             'store_name': self.store_name,
             'model_name': self.model_name
         }
-        
+
         # Add performance metrics if available
         if self.performance_stats:
             stats['performance'] = self.performance_stats
-            
+
         return stats
-    

app/core/exceptions.py

@@ -0,0 +1,201 @@
+#!/usr/bin/env python3
+"""
+Core Exception Classes
+
+Centralized exception definitions for the application.
+This module provides clean exception classes without
+depending on UI or external frameworks.
+"""
+
+
+class AppException(Exception):
+    """Base exception class for application-specific errors"""
+
+    def __init__(self, message: str, user_message: str = None, recovery_hint: str = None):
+        self.message = message
+        self.user_message = user_message or message
+        self.recovery_hint = recovery_hint
+        super().__init__(message)
+
+
+class ValidationError(AppException):
+    """Error for input validation failures"""
+    pass
+
+
+class ProcessingError(AppException):
+    """Error for document processing failures"""
+    pass
+
+
+class AIError(AppException):
+    """Error for AI service failures"""
+    pass
+
+
+class ConfigError(AppException):
+    """Error for configuration issues"""
+    pass
+
+
+class FileOperationError(AppException):
+    """Error for file operation failures"""
+    pass
+
+
+class NetworkError(AppException):
+    """Error for network-related failures"""
+    pass
+
+
+class LLMConnectionError(AIError):
+    """Error for LLM API connection failures"""
+    pass
+
+
+class LLMAuthenticationError(AIError):
+    """Error for LLM API authentication failures"""
+    pass
+
+
+class LLMTimeoutError(AIError):
+    """Error for LLM API timeout failures"""
+    pass
+
+
+class LLMQuotaExceededError(AIError):
+    """Error for LLM API quota/rate limit exceeded"""
+    pass
+
+
+class LLMInvalidResponseError(AIError):
+    """Error for invalid LLM API responses"""
+    pass
+
+
+class DocumentProcessingError(ProcessingError):
+    """Error for document processing failures"""
+    pass
+
+
+class SearchError(AppException):
+    """Error for search operation failures"""
+    pass
+
+
+# Convenience functions for creating exceptions
+def create_validation_error(message: str, recovery_hint: str = None) -> ValidationError:
+    """Create a validation error with consistent formatting"""
+    return ValidationError(
+        message,
+        user_message=f"Validation error: {message}",
+        recovery_hint=recovery_hint or "Please check your input and try again"
+    )
+
+
+def create_processing_error(message: str, recovery_hint: str = None) -> ProcessingError:
+    """Create a processing error with consistent formatting"""
+    return ProcessingError(
+        message,
+        user_message=f"Processing error: {message}",
+        recovery_hint=recovery_hint or "Please check your files and try again"
+    )
+
+
+def create_ai_error(message: str, recovery_hint: str = None) -> AIError:
+    """Create an AI error with consistent formatting"""
+    return AIError(
+        message,
+        user_message=f"AI service error: {message}",
+        recovery_hint=recovery_hint or "Please check your API key and try again"
+    )
+
+
+def create_config_error(message: str, recovery_hint: str = None) -> ConfigError:
+    """Create a configuration error with consistent formatting"""
+    return ConfigError(
+        message,
+        user_message=f"Configuration error: {message}",
+        recovery_hint=recovery_hint or "Please check your configuration and environment variables"
+    )
+
+
+def create_file_error(message: str, recovery_hint: str = None) -> FileOperationError:
+    """Create a file operation error with consistent formatting"""
+    return FileOperationError(
+        message,
+        user_message=f"File error: {message}",
+        recovery_hint=recovery_hint or "Please check file permissions and paths"
+    )
+
+
+def create_network_error(message: str, recovery_hint: str = None) -> NetworkError:
+    """Create a network error with consistent formatting"""
+    return NetworkError(
+        message,
+        user_message=f"Network error: {message}",
+        recovery_hint=recovery_hint or "Please check your internet connection and try again"
+    )
+
+
+def create_llm_connection_error(message: str, recovery_hint: str = None) -> LLMConnectionError:
+    """Create an LLM connection error with consistent formatting"""
+    return LLMConnectionError(
+        message,
+        user_message=f"AI service connection error: {message}",
+        recovery_hint=recovery_hint or "Please check your internet connection and try again"
+    )
+
+
+def create_llm_authentication_error(message: str, recovery_hint: str = None) -> LLMAuthenticationError:
+    """Create an LLM authentication error with consistent formatting"""
+    return LLMAuthenticationError(
+        message,
+        user_message=f"AI service authentication error: {message}",
+        recovery_hint=recovery_hint or "Please check your API key and try again"
+    )
+
+
+def create_llm_timeout_error(message: str, recovery_hint: str = None) -> LLMTimeoutError:
+    """Create an LLM timeout error with consistent formatting"""
+    return LLMTimeoutError(
+        message,
+        user_message=f"AI service timeout: {message}",
+        recovery_hint=recovery_hint or "Please try again in a few moments"
+    )
+
+
+def create_llm_quota_error(message: str, recovery_hint: str = None) -> LLMQuotaExceededError:
+    """Create an LLM quota exceeded error with consistent formatting"""
+    return LLMQuotaExceededError(
+        message,
+        user_message=f"AI service quota exceeded: {message}",
+        recovery_hint=recovery_hint or "Please check your API usage limits and try again later"
+    )
+
+
+def create_llm_invalid_response_error(message: str, recovery_hint: str = None) -> LLMInvalidResponseError:
+    """Create an LLM invalid response error with consistent formatting"""
+    return LLMInvalidResponseError(
+        message,
+        user_message=f"AI service returned invalid response: {message}",
+        recovery_hint=recovery_hint or "Please try again or contact support if the issue persists"
+    )
+
+
+def create_document_processing_error(message: str, recovery_hint: str = None) -> DocumentProcessingError:
+    """Create a document processing error with consistent formatting"""
+    return DocumentProcessingError(
+        message,
+        user_message=f"Document processing error: {message}",
+        recovery_hint=recovery_hint or "Please check your document format and try again"
+    )
+
+
+def create_search_error(message: str, recovery_hint: str = None) -> SearchError:
+    """Create a search error with consistent formatting"""
+    return SearchError(
+        message,
+        user_message=f"Search error: {message}",
+        recovery_hint=recovery_hint or "Please try adjusting your search terms"
+    )

app/core/knowledge_graph.py

@@ -0,0 +1,639 @@
+#!/usr/bin/env python3
+"""
+Knowledge Graph Module
+
+This module provides efficient loading and querying of pre-computed knowledge graphs
+in Streamlit applications. It's designed to work with graphs generated by the
+build_knowledge_graphs.py script.
+
+Key features:
+- Fast graph loading with caching
+- Rich query interface for graph exploration
+- Integration with existing document processor workflow
+- Memory-efficient graph operations
+"""
+
+import pickle
+import json
+import numpy as np
+from pathlib import Path
+from typing import Dict, List, Any, Optional, Set, Tuple
+from datetime import datetime
+import streamlit as st
+
+import networkx as nx
+from sklearn.metrics.pairwise import cosine_similarity
+from app.core.config import get_config
+from app.core.logging import logger
+
+class KnowledgeGraphManager:
+    """
+    Manages loading and querying of knowledge graphs for due diligence analysis.
+    
+    This class provides a clean interface for working with pre-computed knowledge
+    graphs in Streamlit applications, with efficient caching and query capabilities.
+    """
+    
+    def __init__(self, store_name: str):
+        """
+        Initialize the knowledge graph manager for a specific company.
+        
+        Args:
+            store_name: The company store name (matches FAISS index name)
+        """
+        self.store_name = store_name
+        self.graph: Optional[nx.MultiDiGraph] = None
+        self.metadata: Optional[Dict[str, Any]] = None
+        self.entities: Optional[Dict[str, List[Dict]]] = None
+        self.document_processor = None  # Will be loaded on-demand for semantic search
+        self._config = get_config()
+        
+    @st.cache_data(ttl=3600)  # Cache for 1 hour
+    def load_graph(_self) -> bool:
+        """
+        Load the knowledge graph from disk with caching.
+        
+        Returns:
+            bool: True if graph was loaded successfully, False otherwise
+        """
+        try:
+            graphs_dir = _self._config.paths['faiss_dir'] / 'knowledge_graphs'
+            
+            # Load main graph
+            graph_file = graphs_dir / f"{_self.store_name}_knowledge_graph.pkl"
+            if not graph_file.exists():
+                logger.warning(f"Knowledge graph not found: {graph_file}")
+                return False
+            
+            with open(graph_file, 'rb') as f:
+                _self.graph = pickle.load(f)
+            
+            # Load metadata
+            metadata_file = graphs_dir / f"{_self.store_name}_graph_metadata.json"
+            if metadata_file.exists():
+                with open(metadata_file, 'r') as f:
+                    _self.metadata = json.load(f)
+            
+            # Load entities
+            entities_file = graphs_dir / f"{_self.store_name}_entities.json"
+            if entities_file.exists():
+                with open(entities_file, 'r') as f:
+                    _self.entities = json.load(f)
+            
+            logger.info(f"Loaded knowledge graph for {_self.store_name}: "
+                       f"{len(_self.graph.nodes())} nodes, {len(_self.graph.edges())} edges")
+            return True
+            
+        except Exception as e:
+            logger.error(f"Failed to load knowledge graph for {_self.store_name}: {e}")
+            return False
+    
+    def is_available(self) -> bool:
+        """Check if knowledge graph is available and loaded"""
+        return self.graph is not None and len(self.graph.nodes()) > 0
+    
+    def get_summary_stats(self) -> Dict[str, Any]:
+        """Get summary statistics about the knowledge graph"""
+        if not self.is_available():
+            return {}
+        
+        stats = {
+            'num_entities': len(self.graph.nodes()),
+            'num_relationships': len(self.graph.edges()),
+            'entity_types': {},
+            'relationship_types': {},
+            'created_at': self.metadata.get('created_at') if self.metadata else None
+        }
+        
+        # Count entity types
+        for node in self.graph.nodes():
+            node_type = self.graph.nodes[node].get('type', 'unknown')
+            stats['entity_types'][node_type] = stats['entity_types'].get(node_type, 0) + 1
+        
+        # Count relationship types
+        for _, _, edge_data in self.graph.edges(data=True):
+            rel_type = edge_data.get('relationship', 'unknown')
+            stats['relationship_types'][rel_type] = stats['relationship_types'].get(rel_type, 0) + 1
+        
+        return stats
+    
+    def search_entities(self, query: str, entity_type: Optional[str] = None, limit: int = 10) -> List[Dict[str, Any]]:
+        """
+        Search for entities by name or content.
+        
+        Args:
+            query: Search query string
+            entity_type: Filter by entity type (companies, people, etc.)
+            limit: Maximum number of results
+            
+        Returns:
+            List of matching entities with metadata
+        """
+        if not self.is_available():
+            return []
+        
+        query_lower = query.lower()
+        results = []
+        
+        for node in self.graph.nodes():
+            node_data = self.graph.nodes[node]
+            node_name = node_data.get('name', '').lower()
+            node_type = node_data.get('type', '')
+            
+            # Filter by type if specified
+            if entity_type and node_type != entity_type:
+                continue
+            
+            # Check if query matches name or context
+            if query_lower in node_name:
+                score = 1.0 if query_lower == node_name else 0.8
+                
+                results.append({
+                    'node_id': node,
+                    'name': node_data.get('name', ''),
+                    'type': node_type,
+                    'score': score,
+                    'sources': node_data.get('sources', ''),
+                    'document_type': node_data.get('document_type', 'unknown'),
+                    'context_samples': node_data.get('context_samples', [])[:2]  # Limit context
+                })
+        
+        # Sort by score and limit results
+        results.sort(key=lambda x: x['score'], reverse=True)
+        return results[:limit]
+    
+    def get_entity_relationships(self, entity_name: str) -> Dict[str, List[Dict[str, Any]]]:
+        """
+        Get all relationships for a specific entity.
+        
+        Args:
+            entity_name: Name of the entity to find relationships for
+            
+        Returns:
+            Dictionary with 'incoming' and 'outgoing' relationship lists
+        """
+        if not self.is_available():
+            return {'incoming': [], 'outgoing': []}
+        
+        # Find matching nodes
+        matching_nodes = []
+        for node in self.graph.nodes():
+            if entity_name.lower() in self.graph.nodes[node].get('name', '').lower():
+                matching_nodes.append(node)
+        
+        if not matching_nodes:
+            return {'incoming': [], 'outgoing': []}
+        
+        relationships = {'incoming': [], 'outgoing': []}
+        
+        for node in matching_nodes:
+            # Outgoing relationships
+            for _, target, edge_data in self.graph.out_edges(node, data=True):
+                relationships['outgoing'].append({
+                    'target': self.graph.nodes[target].get('name', target),
+                    'target_type': self.graph.nodes[target].get('type', 'unknown'),
+                    'relationship': edge_data.get('relationship', 'unknown'),
+                    'source_document': edge_data.get('source_document', ''),
+                    'context': edge_data.get('context', '')[:200],  # Truncate context
+                    'confidence': edge_data.get('confidence', 0.0)
+                })
+            
+            # Incoming relationships
+            for source, _, edge_data in self.graph.in_edges(node, data=True):
+                relationships['incoming'].append({
+                    'source': self.graph.nodes[source].get('name', source),
+                    'source_type': self.graph.nodes[source].get('type', 'unknown'),
+                    'relationship': edge_data.get('relationship', 'unknown'),
+                    'source_document': edge_data.get('source_document', ''),
+                    'context': edge_data.get('context', '')[:200],  # Truncate context
+                    'confidence': edge_data.get('confidence', 0.0)
+                })
+        
+        return relationships
+    
+    def find_paths(self, source_entity: str, target_entity: str, max_length: int = 3) -> List[List[str]]:
+        """
+        Find paths between two entities in the knowledge graph.
+        
+        Args:
+            source_entity: Starting entity name
+            target_entity: Target entity name
+            max_length: Maximum path length to search
+            
+        Returns:
+            List of paths (each path is a list of entity names)
+        """
+        if not self.is_available():
+            return []
+        
+        # Find matching nodes
+        source_nodes = [n for n in self.graph.nodes() 
+                       if source_entity.lower() in self.graph.nodes[n].get('name', '').lower()]
+        target_nodes = [n for n in self.graph.nodes() 
+                       if target_entity.lower() in self.graph.nodes[n].get('name', '').lower()]
+        
+        if not source_nodes or not target_nodes:
+            return []
+        
+        paths = []
+        for source_node in source_nodes:
+            for target_node in target_nodes:
+                if source_node == target_node:
+                    continue
+                
+                try:
+                    # Find all simple paths up to max_length
+                    simple_paths = list(nx.all_simple_paths(
+                        self.graph, source_node, target_node, cutoff=max_length
+                    ))
+                    
+                    # Convert node IDs to entity names
+                    for path in simple_paths[:5]:  # Limit to 5 paths per pair
+                        entity_path = [self.graph.nodes[node].get('name', node) for node in path]
+                        paths.append(entity_path)
+                        
+                except nx.NetworkXNoPath:
+                    continue
+        
+        return paths[:10]  # Return max 10 paths total
+    
+    def get_central_entities(self, limit: int = 10) -> List[Dict[str, Any]]:
+        """
+        Get the most central/important entities in the graph.
+        
+        Args:
+            limit: Maximum number of entities to return
+            
+        Returns:
+            List of entities with centrality scores
+        """
+        if not self.is_available() or len(self.graph.nodes()) < 2:
+            return []
+        
+        try:
+            # Calculate degree centrality
+            centrality = nx.degree_centrality(self.graph)
+            
+            # Get top central entities
+            top_entities = sorted(centrality.items(), key=lambda x: x[1], reverse=True)[:limit]
+            
+            results = []
+            for node, score in top_entities:
+                node_data = self.graph.nodes[node]
+                results.append({
+                    'name': node_data.get('name', ''),
+                    'type': node_data.get('type', 'unknown'),
+                    'centrality_score': round(score, 3),
+                    'num_connections': len(list(self.graph.neighbors(node))),
+                    'sources': node_data.get('sources', '')
+                })
+            
+            return results
+            
+        except Exception as e:
+            logger.error(f"Error calculating centrality: {e}")
+            return []
+    
+    def get_entity_clusters(self) -> List[List[str]]:
+        """
+        Find clusters of related entities using community detection.
+        
+        Returns:
+            List of clusters (each cluster is a list of entity names)
+        """
+        if not self.is_available() or len(self.graph.nodes()) < 3:
+            return []
+        
+        try:
+            # Convert to undirected graph for community detection
+            undirected = self.graph.to_undirected()
+            
+            # Use simple connected components as clusters
+            components = list(nx.connected_components(undirected))
+            
+            clusters = []
+            for component in components:
+                if len(component) > 1:  # Only include clusters with multiple entities
+                    cluster_names = [self.graph.nodes[node].get('name', node) for node in component]
+                    clusters.append(cluster_names)
+            
+            # Sort clusters by size
+            clusters.sort(key=len, reverse=True)
+            return clusters[:5]  # Return top 5 clusters
+            
+        except Exception as e:
+            logger.error(f"Error finding clusters: {e}")
+            return []
+    
+    def export_graph_data(self) -> Dict[str, Any]:
+        """
+        Export graph data for visualization or further analysis.
+        
+        Returns:
+            Dictionary with nodes and edges data suitable for visualization
+        """
+        if not self.is_available():
+            return {'nodes': [], 'edges': []}
+        
+        # Export nodes
+        nodes = []
+        for node in self.graph.nodes():
+            node_data = self.graph.nodes[node]
+            nodes.append({
+                'id': node,
+                'name': node_data.get('name', ''),
+                'type': node_data.get('type', 'unknown'),
+                'sources': node_data.get('sources', ''),
+                'document_type': node_data.get('document_type', 'unknown')
+            })
+        
+        # Export edges
+        edges = []
+        for source, target, edge_data in self.graph.edges(data=True):
+            edges.append({
+                'source': source,
+                'target': target,
+                'relationship': edge_data.get('relationship', 'unknown'),
+                'source_document': edge_data.get('source_document', ''),
+                'confidence': edge_data.get('confidence', 0.0)
+            })
+        
+        return {
+            'nodes': nodes,
+            'edges': edges,
+            'metadata': self.metadata or {}
+        }
+    
+    def _load_document_processor(self):
+        """Load document processor for semantic search capabilities"""
+        if self.document_processor is None:
+            try:
+                from app.core.utils import create_document_processor
+                self.document_processor = create_document_processor(store_name=self.store_name)
+                if not self.document_processor.vector_store:
+                    logger.warning(f"No FAISS vector store available for {self.store_name}")
+                    self.document_processor = None
+            except Exception as e:
+                logger.error(f"Failed to load document processor for {self.store_name}: {e}")
+                self.document_processor = None
+    
+    def semantic_search_entities(self, query: str, limit: int = 10, similarity_threshold: float = 0.3) -> List[Dict[str, Any]]:
+        """
+        Perform semantic search on entities using FAISS embeddings.
+        
+        This method finds entities whose source contexts are semantically similar
+        to the query, providing more intelligent search than simple text matching.
+        
+        Args:
+            query: Natural language query
+            limit: Maximum number of results
+            similarity_threshold: Minimum similarity score to include
+            
+        Returns:
+            List of entities with similarity scores and context
+        """
+        if not self.is_available():
+            return []
+        
+        # Load document processor if not already loaded
+        self._load_document_processor()
+        if not self.document_processor or not self.document_processor.vector_store:
+            logger.warning("Semantic search not available - falling back to text search")
+            return self.search_entities(query, limit=limit)
+        
+        try:
+            # Perform semantic search on FAISS index
+            relevant_docs = self.document_processor.vector_store.similarity_search_with_score(
+                query, k=min(50, limit * 5)  # Get more candidates for filtering
+            )
+            
+            # Map document chunks back to entities
+            entity_matches = []
+            seen_entities = set()
+            
+            for doc, score in relevant_docs:
+                if score < similarity_threshold:
+                    continue
+                
+                # Find entities that originated from this document chunk
+                chunk_id = doc.metadata.get('chunk_id', '')
+                doc_source = doc.metadata.get('source', '')
+                
+                # Search for entities that came from this chunk/document
+                for node in self.graph.nodes():
+                    node_data = self.graph.nodes[node]
+                    entity_sources = node_data.get('sources', '')
+                    
+                    # Check if entity came from this document
+                    if (doc_source and doc_source in entity_sources) or (chunk_id and chunk_id in str(node_data.get('context_samples', []))):
+                        entity_key = f"{node_data.get('name', '')}_{node_data.get('type', '')}"
+                        
+                        if entity_key not in seen_entities:
+                            seen_entities.add(entity_key)
+                            entity_matches.append({
+                                'node_id': node,
+                                'name': node_data.get('name', ''),
+                                'type': node_data.get('type', 'unknown'),
+                                'similarity_score': 1.0 - score,  # Convert distance to similarity
+                                'sources': entity_sources,
+                                'document_type': node_data.get('document_type', 'unknown'),
+                                'context_samples': node_data.get('context_samples', [])[:2],
+                                'matching_context': doc.page_content[:300]  # Show relevant context
+                            })
+                            
+                            if len(entity_matches) >= limit:
+                                break
+                
+                if len(entity_matches) >= limit:
+                    break
+            
+            # Sort by similarity score
+            entity_matches.sort(key=lambda x: x['similarity_score'], reverse=True)
+            return entity_matches[:limit]
+            
+        except Exception as e:
+            logger.error(f"Semantic search failed: {e}")
+            # Fallback to regular text search
+            return self.search_entities(query, limit=limit)
+    
+    def find_related_entities_by_context(self, entity_name: str, limit: int = 5) -> List[Dict[str, Any]]:
+        """
+        Find entities related to the given entity based on semantic similarity of their contexts.
+        
+        Args:
+            entity_name: Name of the reference entity
+            limit: Maximum number of related entities to return
+            
+        Returns:
+            List of related entities with similarity scores
+        """
+        if not self.is_available():
+            return []
+        
+        # Find the reference entity
+        reference_entities = [n for n in self.graph.nodes() 
+                            if entity_name.lower() in self.graph.nodes[n].get('name', '').lower()]
+        
+        if not reference_entities:
+            return []
+        
+        # Load document processor
+        self._load_document_processor()
+        if not self.document_processor or not self.document_processor.vector_store:
+            return []
+        
+        try:
+            # Get context samples from the reference entity
+            reference_node = reference_entities[0]
+            reference_data = self.graph.nodes[reference_node]
+            context_samples = reference_data.get('context_samples', [])
+            
+            if not context_samples:
+                return []
+            
+            # Use the first context sample as a query
+            query_context = context_samples[0][:500]  # Limit context length
+            
+            # Find semantically similar contexts
+            similar_docs = self.document_processor.vector_store.similarity_search_with_score(
+                query_context, k=20
+            )
+            
+            # Map back to entities
+            related_entities = []
+            seen_entities = {reference_data.get('name', '')}
+            
+            for doc, score in similar_docs:
+                doc_source = doc.metadata.get('source', '')
+                
+                # Find entities from this document
+                for node in self.graph.nodes():
+                    if node == reference_node:
+                        continue
+                    
+                    node_data = self.graph.nodes[node]
+                    entity_name_node = node_data.get('name', '')
+                    entity_sources = node_data.get('sources', '')
+                    
+                    if (entity_name_node not in seen_entities and 
+                        doc_source and doc_source in entity_sources):
+                        
+                        seen_entities.add(entity_name_node)
+                        related_entities.append({
+                            'name': entity_name_node,
+                            'type': node_data.get('type', 'unknown'),
+                            'similarity_score': 1.0 - score,
+                            'sources': entity_sources,
+                            'context_samples': node_data.get('context_samples', [])[:1],
+                            'relationship_reason': 'Semantic context similarity'
+                        })
+                        
+                        if len(related_entities) >= limit:
+                            break
+                
+                if len(related_entities) >= limit:
+                    break
+            
+            # Sort by similarity
+            related_entities.sort(key=lambda x: x['similarity_score'], reverse=True)
+            return related_entities[:limit]
+            
+        except Exception as e:
+            logger.error(f"Context-based entity search failed: {e}")
+            return []
+    
+    def semantic_path_search(self, query: str, max_paths: int = 5) -> List[Dict[str, Any]]:
+        """
+        Find paths in the graph that are semantically relevant to a query.
+        
+        Args:
+            query: Natural language description of what to find
+            max_paths: Maximum number of paths to return
+            
+        Returns:
+            List of paths with relevance scores
+        """
+        if not self.is_available():
+            return []
+        
+        # First, find entities semantically related to the query
+        relevant_entities = self.semantic_search_entities(query, limit=10)
+        
+        if len(relevant_entities) < 2:
+            return []
+        
+        # Find interesting paths between the most relevant entities
+        paths_found = []
+        
+        for i, entity1 in enumerate(relevant_entities[:5]):  # Limit to top 5 for performance
+            for entity2 in relevant_entities[i+1:]:
+                try:
+                    # Find paths between these entities
+                    paths = self.find_paths(entity1['name'], entity2['name'], max_length=3)
+                    
+                    for path in paths[:2]:  # Limit paths per pair
+                        # Calculate path relevance based on entity similarity scores
+                        path_score = (entity1['similarity_score'] + entity2['similarity_score']) / 2
+                        
+                        paths_found.append({
+                            'path': path,
+                            'relevance_score': path_score,
+                            'start_entity': entity1['name'],
+                            'end_entity': entity2['name'],
+                            'query_relevance': f"Related to: {query}",
+                            'path_length': len(path) - 1
+                        })
+                        
+                        if len(paths_found) >= max_paths:
+                            break
+                
+                except Exception as e:
+                    logger.debug(f"Path finding failed between {entity1['name']} and {entity2['name']}: {e}")
+                    continue
+                
+                if len(paths_found) >= max_paths:
+                    break
+            
+            if len(paths_found) >= max_paths:
+                break
+        
+        # Sort by relevance score
+        paths_found.sort(key=lambda x: x['relevance_score'], reverse=True)
+        return paths_found[:max_paths]
+
+@st.cache_data(ttl=3600)
+def get_available_knowledge_graphs() -> List[str]:
+    """
+    Get list of available knowledge graphs.
+    
+    Returns:
+        List of store names that have knowledge graphs available
+    """
+    try:
+        config = get_config()
+        graphs_dir = config.paths['faiss_dir'] / 'knowledge_graphs'
+        
+        if not graphs_dir.exists():
+            return []
+        
+        # Find all knowledge graph files
+        graph_files = list(graphs_dir.glob("*_knowledge_graph.pkl"))
+        store_names = [f.stem.replace('_knowledge_graph', '') for f in graph_files]
+        
+        return sorted(store_names)
+        
+    except Exception as e:
+        logger.error(f"Error getting available knowledge graphs: {e}")
+        return []
+
+def create_knowledge_graph_manager(store_name: str) -> KnowledgeGraphManager:
+    """
+    Factory function to create a knowledge graph manager.
+    
+    Args:
+        store_name: Company store name
+        
+    Returns:
+        Configured KnowledgeGraphManager instance
+    """
+    return KnowledgeGraphManager(store_name)

app/core/logging.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+"""
+Logging Configuration Module
+
+Provides consistent logging setup for the application.
+This replaces the old src-based logging with a cleaner, app-specific solution.
+"""
+
+import logging
+import sys
+from pathlib import Path
+from logging.handlers import RotatingFileHandler
+
+
+def configure_langchain_logging(log_level: str = "WARNING") -> None:
+    """
+    Configure LangChain library logging levels to reduce verbosity.
+
+    Args:
+        log_level: Logging level for LangChain modules (default: WARNING)
+    """
+    langchain_modules = [
+        "langchain",
+        "langchain_core",
+        "langchain_community",
+        "langchain_huggingface"
+    ]
+
+    level = getattr(logging, log_level.upper())
+    for module in langchain_modules:
+        logging.getLogger(module).setLevel(level)
+
+
+def setup_logging(
+    name: str = "dd_poc",
+    log_level: str = "INFO",
+    log_file: str = None
+) -> logging.Logger:
+    """
+    Set up standard Python logging with rotating file handler
+
+    Args:
+        name: Logger name
+        log_level: Logging level
+        log_file: Optional log file path
+
+    Returns:
+        Configured logger instance
+    """
+    logger = logging.getLogger(name)
+
+    # Avoid duplicate setup if logger already has handlers
+    if logger.handlers:
+        return logger
+
+    logger.setLevel(getattr(logging, log_level.upper()))
+
+    # Console handler
+    console_handler = logging.StreamHandler(sys.stdout)
+    console_formatter = logging.Formatter(
+        '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+    )
+    console_handler.setFormatter(console_formatter)
+    logger.addHandler(console_handler)
+
+    # Rotating file handler (if possible)
+    if log_file or True:  # Always try to set up file logging
+        try:
+            log_dir = Path(".logs")
+            log_dir.mkdir(exist_ok=True)
+
+            if not log_file:
+                log_file = log_dir / f"dd_poc_{Path.cwd().name}.log"
+
+            # Use RotatingFileHandler for better log management
+            file_handler = RotatingFileHandler(
+                log_file,
+                maxBytes=10 * 1024 * 1024,  # 10MB
+                backupCount=5
+            )
+            file_formatter = logging.Formatter(
+                '%(asctime)s - %(name)s - %(levelname)s - %(funcName)s:%(lineno)d - %(message)s'
+            )
+            file_handler.setFormatter(file_formatter)
+            logger.addHandler(file_handler)
+        except Exception:
+            # File logging not available (e.g., on Streamlit Cloud)
+            pass
+
+    return logger
+
+
+# Global logger instance
+logger = setup_logging()

app/core/model_cache.py

@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+"""
+Model Cache Manager
+
+Provides global caching for HuggingFace models to prevent re-downloads
+across multiple instances and sessions.
+"""
+
+import logging
+from typing import Optional
+from pathlib import Path
+from langchain_huggingface import HuggingFaceEmbeddings
+from sentence_transformers import CrossEncoder
+
+from app.core.logging import logger
+
+# Optional accelerate import
+try:
+    from accelerate import Accelerator
+    ACCELERATE_AVAILABLE = True
+except ImportError:
+    ACCELERATE_AVAILABLE = False
+    Accelerator = None
+
+# Global model cache
+_EMBEDDINGS_CACHE = {}
+_CROSS_ENCODER_CACHE = {}
+
+# Local models directory
+_MODELS_DIR = Path(__file__).parent.parent.parent / "models"
+
+def _get_local_model_path(model_name: str) -> Optional[Path]:
+    """
+    Get local path for a model if it exists.
+
+    Args:
+        model_name: HuggingFace model name
+
+    Returns:
+        Path to local model directory or None if not found
+    """
+    if "/" in model_name:
+        # Handle different model name formats
+        if model_name.startswith("sentence-transformers/"):
+            # For sentence transformers: sentence-transformers/all-mpnet-base-v2
+            model_short_name = model_name.split("/")[-1]
+            local_path = _MODELS_DIR / "sentence_transformers" / model_short_name
+        elif model_name.startswith("cross-encoder/"):
+            # For cross encoders: cross-encoder/ms-marco-MiniLM-L-6-v2
+            model_short_name = model_name.split("/")[-1]
+            local_path = _MODELS_DIR / "cross_encoder" / model_short_name
+        else:
+            # Fallback for other models
+            model_short_name = model_name.split("/")[-1]
+            local_path = _MODELS_DIR / model_short_name
+
+        if local_path.exists():
+            return local_path
+
+    return None
+
+def get_cached_embeddings(model_name: str = "sentence-transformers/all-mpnet-base-v2") -> HuggingFaceEmbeddings:
+    """
+    Get cached HuggingFace embeddings model with accelerate optimization.
+
+    Creates the model only once and reuses it across all instances.
+    Uses local models directory if available, otherwise downloads from HuggingFace.
+    Automatically uses GPU if available via accelerate.
+    """
+    if model_name not in _EMBEDDINGS_CACHE:
+        # Check for local model first
+        local_path = _get_local_model_path(model_name)
+        if local_path:
+            logger.info(f"Using local embeddings model: {local_path}")
+            embeddings = HuggingFaceEmbeddings(model_name=str(local_path))
+        else:
+            logger.info(f"Downloading embeddings model: {model_name}")
+            embeddings = HuggingFaceEmbeddings(model_name=model_name)
+
+        # Optimize device placement with accelerate if available
+        if ACCELERATE_AVAILABLE:
+            try:
+                accelerator = Accelerator()
+                logger.info(f"Embeddings model optimized for device: {accelerator.device}")
+                # Accelerate will automatically handle device placement
+            except Exception as e:
+                logger.warning(f"Failed to optimize embeddings with accelerate: {e}")
+
+        _EMBEDDINGS_CACHE[model_name] = embeddings
+    else:
+        logger.debug(f"Using cached embeddings model: {model_name}")
+
+    return _EMBEDDINGS_CACHE[model_name]
+
+def get_cached_cross_encoder(model_name: str = 'cross-encoder/ms-marco-MiniLM-L-6-v2') -> CrossEncoder:
+    """
+    Get cached cross-encoder model.
+
+    Creates the model only once and reuses it across all instances.
+    Uses local models directory if available, otherwise downloads from HuggingFace.
+    """
+    if model_name not in _CROSS_ENCODER_CACHE:
+        # Check for local model first
+        local_path = _get_local_model_path(model_name)
+        if local_path:
+            logger.info(f"Using local cross-encoder model: {local_path}")
+            _CROSS_ENCODER_CACHE[model_name] = CrossEncoder(str(local_path))
+        else:
+            logger.info(f"Downloading cross-encoder model: {model_name}")
+            _CROSS_ENCODER_CACHE[model_name] = CrossEncoder(model_name)
+    else:
+        logger.debug(f"Using cached cross-encoder model: {model_name}")
+
+    return _CROSS_ENCODER_CACHE[model_name]
+
+def clear_model_cache():
+    """
+    Clear all cached models.
+
+    Useful for memory management or testing.
+    """
+    _EMBEDDINGS_CACHE.clear()
+    _CROSS_ENCODER_CACHE.clear()
+    logger.info("Model cache cleared")

app/core/parsers.py

@@ -0,0 +1,155 @@
+#!/usr/bin/env python3
+"""
+LLM-based parsing functions for due diligence documents.
+
+This module provides modern structured output parsing using Pydantic models
+to ensure reliable, type-safe parsing of LLM responses.
+"""
+
+from typing import Dict, List
+from app.core.logging import logger
+
+
+def parse_checklist(checklist_text: str, llm) -> Dict:
+    """
+    Parse markdown checklist using Pydantic structured output.
+    
+    This approach uses LangChain's PydanticOutputParser to ensure the LLM
+    returns properly structured data that matches our expected format.
+    
+    Args:
+        checklist_text: The raw checklist text to parse
+        llm: LLM instance to use for parsing
+        
+    Returns:
+        Dictionary with categories and their items
+        
+    Raises:
+        RuntimeError: If LLM is not available or parsing fails
+        ValueError: If llm parameter is not provided
+    """
+    if llm is None:
+        raise ValueError("LLM parameter is required")
+    
+    try:
+        from langchain_core.output_parsers import PydanticOutputParser
+        from app.ai.processing_pipeline import StructuredChecklist
+        from app.ai.prompts import get_checklist_parsing_prompt
+        
+        # Set up structured output parser
+        parser = PydanticOutputParser(pydantic_object=StructuredChecklist)
+        
+        # Use centralized prompt from prompts.py (avoid duplication)
+        prompt = get_checklist_parsing_prompt()
+        
+        # Format the prompt with the checklist text and format instructions
+        formatted_prompt = prompt.format_messages(
+            checklist_text=checklist_text,
+            format_instructions=parser.get_format_instructions()
+        )
+        
+        # Get LLM response
+        logger.info(f"Sending checklist to LLM for parsing (length: {len(checklist_text)} chars)")
+        llm_response = llm.invoke(formatted_prompt)
+        logger.debug(f"LLM response length: {len(llm_response.content)} chars")
+        
+        # Parse the response using the Pydantic parser
+        result = parser.parse(llm_response.content)
+        
+        # Convert Pydantic model to expected dictionary format
+        categories_dict = {}
+        for key, category in result.categories.items():
+            categories_dict[key] = {
+                'name': category.name,
+                'items': [
+                    {
+                        'text': item.text,
+                        'original': item.original
+                    }
+                    for item in category.items
+                ]
+            }
+        
+        logger.info(f"Successfully parsed {len(categories_dict)} categories: {list(categories_dict.keys())}")
+        return categories_dict
+        
+    except Exception as e:
+        raise RuntimeError(f"Structured parsing failed: {str(e)}")
+
+
+def parse_questions(questions_text: str, llm) -> List[Dict]:
+    """
+    Parse markdown questions using Pydantic structured output.
+    
+    Args:
+        questions_text: The raw questions text to parse
+        llm: LLM instance to use for parsing
+        
+    Returns:
+        List of dictionaries with question data
+        
+    Raises:
+        RuntimeError: If LLM is not available or parsing fails
+        ValueError: If llm parameter is not provided
+    """
+    if llm is None:
+        raise ValueError("LLM parameter is required")
+    
+    try:
+        from langchain_core.output_parsers import PydanticOutputParser
+        from langchain_core.prompts import ChatPromptTemplate
+        from langchain_core.messages import SystemMessage, HumanMessage
+        from app.ai.processing_pipeline import StructuredQuestions
+        
+        # Set up structured output parser
+        parser = PydanticOutputParser(pydantic_object=StructuredQuestions)
+        
+        # Create prompt with format instructions
+        from langchain_core.prompts import HumanMessagePromptTemplate
+        prompt = ChatPromptTemplate.from_messages([
+            SystemMessage(content="""
+You are a document parser. Parse the due diligence questions document into the EXACT JSON format specified.
+
+CRITICAL:
+- Return ONLY valid JSON, no additional text or explanations
+- Extract categories (like "### A. Category Name")
+- Extract numbered questions within each category
+- Clean up markdown formatting but preserve core text
+- Follow the exact format specified in the format instructions
+
+The output must be valid JSON that can be parsed directly.
+"""),
+            HumanMessagePromptTemplate.from_template("""Parse these questions into the exact JSON format:
+
+{questions_text}
+
+Required JSON schema:
+{format_instructions}
+
+Return only the JSON:""")
+        ])
+        
+        # Format the prompt with the questions text and format instructions
+        formatted_prompt = prompt.format_messages(
+            questions_text=questions_text,
+            format_instructions=parser.get_format_instructions()
+        )
+        
+        # Get LLM response
+        llm_response = llm.invoke(formatted_prompt)
+        
+        # Parse the response using the Pydantic parser
+        result = parser.parse(llm_response.content)
+        
+        # Convert Pydantic model to expected list format
+        return [
+            {
+                'category': question.category,
+                'question': question.question,
+                'id': question.id
+            }
+            for question in result.questions
+        ]
+        
+    except Exception as e:
+        raise RuntimeError(f"Structured parsing failed: {str(e)}")

app/core/performance.py

@@ -0,0 +1,382 @@
+#!/usr/bin/env python3
+"""
+Performance Optimization Module
+
+This module provides performance optimizations using prebuilt libraries:
+- diskcache: Smart caching system
+- joblib: Function result caching
+- httpx: Async HTTP client
+- backoff: Retry logic with exponential backoff
+- psutil: System resource monitoring
+"""
+
+import asyncio
+import hashlib
+import logging
+import time
+from pathlib import Path
+from typing import Dict, List, Any, Optional, Callable, TypeVar, Union
+from functools import wraps
+
+import diskcache
+import joblib
+import httpx
+import backoff
+import psutil
+from tqdm import tqdm
+
+# Optional imports for GPU/CPU optimization
+try:
+    import accelerate
+    ACCELERATE_AVAILABLE = True
+except ImportError:
+    ACCELERATE_AVAILABLE = False
+
+try:
+    import memory_profiler
+    MEMORY_PROFILER_AVAILABLE = True
+except ImportError:
+    MEMORY_PROFILER_AVAILABLE = False
+
+from app.core.config import get_config
+
+logger = logging.getLogger(__name__)
+
+# Type hints
+T = TypeVar('T')
+
+class PerformanceManager:
+    """Central manager for performance optimizations"""
+
+    def __init__(self):
+        self.config = get_config()
+        self._setup_caches()
+        self._setup_clients()
+
+    def _setup_caches(self):
+        """Initialize caching systems"""
+        faiss_dir = self.config.paths['faiss_dir']
+        faiss_dir.mkdir(parents=True, exist_ok=True)
+
+        # Document content cache
+        self.doc_cache = diskcache.Cache(
+            str(faiss_dir / '.doc_cache'),
+            size_limit=500 * 1024 * 1024,  # 500MB
+            eviction_policy='least-recently-used'
+        )
+
+        # Embedding cache
+        self.embedding_cache = diskcache.Cache(
+            str(faiss_dir / '.embedding_cache'),
+            size_limit=2 * 1024 * 1024 * 1024,  # 2GB
+            eviction_policy='least-recently-used'
+        )
+
+        # Joblib memory cache for expensive computations
+        self.memory = joblib.Memory(
+            location=str(faiss_dir / '.joblib_cache'),
+            verbose=0,
+            compress=True
+        )
+
+    def _setup_clients(self):
+        """Initialize HTTP clients"""
+        # Async HTTP client for AI API calls
+        self.http_client = httpx.AsyncClient(
+            timeout=httpx.Timeout(60.0, connect=10.0),
+            limits=httpx.Limits(max_connections=10, max_keepalive_connections=5)
+        )
+
+    @staticmethod
+    def get_file_hash(file_path: Path) -> str:
+        """Calculate SHA256 hash of file content"""
+        hash_sha256 = hashlib.sha256()
+        with open(file_path, 'rb') as f:
+            for chunk in iter(lambda: f.read(4096), b""):
+                hash_sha256.update(chunk)
+        return hash_sha256.hexdigest()
+
+    def cache_document_content(self, file_path: Path, content: str) -> None:
+        """Cache document content with hash-based key"""
+        file_hash = self.get_file_hash(file_path)
+        cache_key = f"doc_content:{file_hash}"
+        self.doc_cache.set(cache_key, content, expire=86400 * 30)  # 30 days
+
+    def get_cached_document_content(self, file_path: Path) -> Optional[str]:
+        """Get cached document content"""
+        file_hash = self.get_file_hash(file_path)
+        cache_key = f"doc_content:{file_hash}"
+        return self.doc_cache.get(cache_key)
+
+    def cache_embeddings(self, text_hash: str, embeddings: List[List[float]]) -> None:
+        """Cache embeddings with content hash"""
+        cache_key = f"embeddings:{text_hash}"
+        self.embedding_cache.set(cache_key, embeddings, expire=86400 * 30)
+
+    def get_cached_embeddings(self, text_hash: str) -> Optional[List[List[float]]]:
+        """Get cached embeddings"""
+        cache_key = f"embeddings:{text_hash}"
+        return self.embedding_cache.get(cache_key)
+
+    @backoff.on_exception(
+        backoff.expo,
+        (httpx.RequestError, httpx.TimeoutException),
+        max_tries=3,
+        jitter=backoff.random_jitter
+    )
+    async def make_api_request(self, url: str, **kwargs) -> httpx.Response:
+        """Make API request with automatic retry logic"""
+        return await self.http_client.request(url=url, **kwargs)
+
+    def monitor_memory_usage(self) -> Dict[str, float]:
+        """Monitor current memory usage"""
+        process = psutil.Process()
+        memory_info = process.memory_info()
+
+        result = {
+            'rss': memory_info.rss / 1024 / 1024,  # MB
+            'vms': memory_info.vms / 1024 / 1024,  # MB
+            'percent': process.memory_percent()
+        }
+
+        # Add GPU memory info if available
+        if ACCELERATE_AVAILABLE:
+            try:
+                import torch
+                if torch.cuda.is_available():
+                    gpu_memory = torch.cuda.get_device_properties(0)
+                    result.update({
+                        'gpu_total': gpu_memory.total_memory / 1024 / 1024 / 1024,  # GB
+                        'gpu_allocated': torch.cuda.memory_allocated(0) / 1024 / 1024 / 1024,  # GB
+                        'gpu_reserved': torch.cuda.memory_reserved(0) / 1024 / 1024 / 1024,  # GB
+                    })
+            except Exception as e:
+                logger.debug(f"Could not get GPU memory info: {e}")
+
+        return result
+
+    def should_gc_collect(self, memory_usage: Dict[str, float]) -> bool:
+        """Determine if garbage collection should be triggered"""
+        return memory_usage['percent'] > 80.0 or memory_usage['rss'] > 2000  # 80% or 2GB
+
+    def cleanup_cache(self) -> Dict[str, int]:
+        """Clean up expired cache entries"""
+        doc_cleaned = self.doc_cache.expire()
+        embedding_cleaned = self.embedding_cache.expire()
+
+        return {
+            'doc_cache_cleaned': doc_cleaned,
+            'embedding_cache_cleaned': embedding_cleaned
+        }
+
+    async def close(self):
+        """Clean up resources"""
+        await self.http_client.aclose()
+        self.doc_cache.close()
+        self.embedding_cache.close()
+
+    def optimize_batch_size(self, available_memory: float, item_size_estimate: float = 0.1) -> int:
+        """Dynamically optimize batch size based on available memory"""
+        # Reserve 20% of memory for overhead
+        usable_memory = available_memory * 0.8
+
+        # Estimate optimal batch size
+        optimal_batch = int(usable_memory / item_size_estimate)
+
+        # Clamp to reasonable bounds
+        return max(1, min(optimal_batch, 1000))
+
+    def get_optimal_device(self) -> str:
+        """Get the optimal device for computations"""
+        if ACCELERATE_AVAILABLE:
+            try:
+                import torch
+                if torch.cuda.is_available():
+                    return 'cuda'
+            except:
+                pass
+        return 'cpu'
+
+    def setup_accelerate(self):
+        """Setup accelerate for optimal performance"""
+        if ACCELERATE_AVAILABLE:
+            try:
+                from accelerate import Accelerator
+                self.accelerator = Accelerator()
+                logger.info(f"Accelerate initialized with device: {self.accelerator.device}")
+                return self.accelerator
+            except Exception as e:
+                logger.warning(f"Failed to initialize accelerate: {e}")
+        return None
+
+
+# Global performance manager instance
+_perf_manager = None
+
+def get_performance_manager() -> PerformanceManager:
+    """Get global performance manager instance"""
+    global _perf_manager
+    if _perf_manager is None:
+        _perf_manager = PerformanceManager()
+    return _perf_manager
+
+
+# Decorators for easy optimization
+def cached_by_content(func: Callable[..., T]) -> Callable[..., T]:
+    """Cache function results based on content hash"""
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        # Generate content hash from relevant arguments
+        content_parts = []
+        for arg in args[1:]:  # Skip self
+            if isinstance(arg, (str, Path)):
+                content_parts.append(str(arg))
+
+        content_hash = hashlib.sha256(
+            '|'.join(content_parts).encode()
+        ).hexdigest()[:16]
+
+        perf_manager = get_performance_manager()
+        cache_key = f"{func.__name__}:{content_hash}"
+
+        # Try cache first
+        result = perf_manager.doc_cache.get(cache_key)
+        if result is not None:
+            logger.debug(f"Cache hit for {func.__name__}")
+            return result
+
+        # Compute and cache
+        result = func(*args, **kwargs)
+        perf_manager.doc_cache.set(cache_key, result, expire=86400 * 7)  # 7 days
+        return result
+
+    return wrapper
+
+
+def memory_cached(func: Callable[..., T]) -> Callable[..., T]:
+    """Cache function results using joblib memory cache"""
+    perf_manager = get_performance_manager()
+    cached_func = perf_manager.memory.cache(func)
+    return cached_func
+
+
+def monitor_performance(func: Callable[..., T]) -> Callable[..., T]:
+    """Monitor function performance and memory usage"""
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        start_time = time.time()
+        perf_manager = get_performance_manager()
+
+        # Memory before
+        mem_before = perf_manager.monitor_memory_usage()
+
+        try:
+            result = func(*args, **kwargs)
+            return result
+        finally:
+            # Memory after
+            mem_after = perf_manager.monitor_memory_usage()
+            duration = time.time() - start_time
+
+            logger.debug(
+                f"{func.__name__}: {duration:.2f}s, "
+                f"Memory: {mem_before['rss']:.1f}MB -> {mem_after['rss']:.1f}MB"
+            )
+
+            # Trigger GC if needed
+            if perf_manager.should_gc_collect(mem_after):
+                import gc
+                gc.collect()
+                logger.debug("Garbage collection triggered")
+
+    return wrapper
+
+
+# Utility functions
+def get_text_hash(text: str) -> str:
+    """Generate hash for text content"""
+    return hashlib.sha256(text.encode()).hexdigest()[:16]
+
+
+def parallel_process(items: List[T], func: Callable[[T], Any],
+                    max_workers: int = 4, desc: str = "Processing") -> List[Any]:
+    """Process items in parallel using ThreadPoolExecutor"""
+    from concurrent.futures import ThreadPoolExecutor, as_completed
+
+    results = []
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        futures = {executor.submit(func, item): item for item in items}
+
+        with tqdm(total=len(items), desc=desc) as pbar:
+            for future in as_completed(futures):
+                result = future.result()
+                results.append(result)
+                pbar.update(1)
+
+    return results
+
+
+def optimize_embedding_batch(texts: List[str], embeddings_model,
+                           batch_size: int = 32) -> List[List[float]]:
+    """Optimize embedding generation with dynamic batching"""
+    perf_manager = get_performance_manager()
+
+    # Get available memory for batch optimization
+    mem_info = perf_manager.monitor_memory_usage()
+    available_memory = mem_info['rss']
+
+    # Dynamically adjust batch size based on memory
+    optimal_batch = perf_manager.optimize_batch_size(available_memory, item_size_estimate=0.001)
+    batch_size = min(batch_size, optimal_batch)
+
+    logger.info(f"Using optimized batch size: {batch_size} (memory: {available_memory:.1f}MB)")
+
+    all_embeddings = []
+
+    # Process in optimized batches
+    for i in range(0, len(texts), batch_size):
+        batch = texts[i:i + batch_size]
+
+        # Monitor memory before processing
+        mem_before = perf_manager.monitor_memory_usage()
+
+        try:
+            # Generate embeddings for this batch
+            batch_embeddings = embeddings_model.embed_documents(batch)
+            all_embeddings.extend(batch_embeddings)
+
+            # Monitor memory after processing
+            mem_after = perf_manager.monitor_memory_usage()
+
+            # Trigger GC if memory usage is high
+            if perf_manager.should_gc_collect(mem_after):
+                import gc
+                gc.collect()
+                logger.debug("GC triggered during embedding generation")
+
+        except Exception as e:
+            logger.error(f"Failed to process embedding batch {i//batch_size}: {e}")
+            # Continue with empty embeddings for this batch
+            all_embeddings.extend([[] for _ in batch])
+
+    return all_embeddings
+
+
+async def gather_with_concurrency(n: int, *tasks):
+    """Run async tasks with controlled concurrency"""
+    semaphore = asyncio.Semaphore(n)
+
+    async def sem_task(task):
+        async with semaphore:
+            return await task
+
+    return await asyncio.gather(*(sem_task(task) for task in tasks))
+
+
+# Cleanup function for graceful shutdown
+async def cleanup_performance_resources():
+    """Clean up performance resources"""
+    global _perf_manager
+    if _perf_manager:
+        await _perf_manager.close()
+        _perf_manager = None

app/core/ranking.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+"""
+Ranking utilities for search results reranking.
+
+This module provides functions for reranking search results using cross-encoder models
+to improve relevance scoring. Separated from search.py to avoid circular imports.
+"""
+
+from typing import Dict, List
+from app.core.logging import logger
+from app.core.model_cache import get_cached_cross_encoder
+
+
+def rerank_results(query: str, candidates: List[Dict]) -> List[Dict]:
+    """
+    Rerank search results using cross-encoder model for improved relevance
+
+    Args:
+        query: The search query
+        candidates: List of candidate documents with 'text', 'score', etc.
+
+    Returns:
+        Reranked list of candidates with updated scores
+    """
+    if not candidates:
+        return candidates
+
+    try:
+        # Get cached cross-encoder model
+        cross_encoder = get_cached_cross_encoder()
+
+        # Prepare input pairs for cross-encoder
+        query_doc_pairs = [(query, candidate['text']) for candidate in candidates]
+
+        # Get cross-encoder scores
+        ce_scores = cross_encoder.predict(query_doc_pairs)
+
+        # Update candidates with reranked scores
+        for i, candidate in enumerate(candidates):
+            candidate['reranked_score'] = float(ce_scores[i])
+            candidate['score'] = float(ce_scores[i])  # Update main score for consistency
+
+        # Sort by reranked score (higher is better for cross-encoder)
+        candidates.sort(key=lambda x: x['reranked_score'], reverse=True)
+
+        logger.info(f"Reranked {len(candidates)} results using cross-encoder")
+        return candidates
+
+    except Exception as e:
+        logger.warning(f"Cross-encoder reranking failed: {e}. Using original scores.")
+        return candidates

app/core/reports.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python3
+"""
+Report generation functions for due diligence analysis.
+"""
+
+from typing import Dict
+
+from app.core.logging import logger
+
+
+def generate_reports_from_cache(checklist_results: Dict, questions_answers: Dict, strategy_text: str, checklist_text: str, questions_text: str) -> Dict:
+    """Generate reports from cached results (placeholder implementation)"""
+    logger.info("Generating reports from cache")
+
+    return {
+        'overview': "Report generated from cached data",
+        'strategic': strategy_text[:500] if strategy_text else "No strategy provided",
+        'checklist_summary': f"Processed {len(checklist_results)} categories",
+        'questions_summary': f"Processed {len(questions_answers)} questions"
+    }
+
+
+def generate_reports(checklist_results: Dict, questions_answers: Dict, strategy_text: str, checklist_text: str, questions_text: str) -> Dict:
+    """Generate comprehensive reports (placeholder implementation)"""
+    logger.info("Generating comprehensive reports")
+
+    return {
+        'overview': "Comprehensive report generated",
+        'strategic': strategy_text[:1000] if strategy_text else "No strategy provided",
+        'checklist_summary': f"Processed {len(checklist_results)} categories with detailed analysis",
+        'questions_summary': f"Processed {len(questions_answers)} questions with detailed answers"
+    }

app/core/search.py

@@ -0,0 +1,773 @@
+#!/usr/bin/env python3
+"""
+Search and analysis functions for document retrieval and ranking.
+"""
+
+# Standard library imports
+from typing import Dict, List
+from pathlib import Path
+
+# Third-party imports for Unicode normalization
+import unidecode
+
+# Third-party imports
+import numpy as np
+from langchain.chains.combine_documents import create_stuff_documents_chain
+from langchain.chains.retrieval import create_retrieval_chain
+from langchain_community.vectorstores import FAISS
+from langchain_core.prompts import PromptTemplate
+
+# Local imports
+from app.core.constants import SIMILARITY_THRESHOLD
+from app.core.document_processor import DocumentProcessor
+from app.core.logging import logger
+from app.core.ranking import rerank_results
+from app.core.sparse_index import load_sparse_index_for_store, BM25Index
+
+
+def search_and_analyze(queries: List[Dict], vector_store: FAISS, llm=None, threshold: float = SIMILARITY_THRESHOLD, search_type: str = 'items', store_name: str = None, session=None) -> Dict:
+    """Unified search function for both checklist items and questions using direct FAISS search for accurate scores"""
+
+    # Create RAG chain if LLM is provided
+    qa_chain = None
+    if llm:
+        retriever = vector_store.as_retriever(
+            search_type="similarity_score_threshold",
+            search_kwargs={"score_threshold": threshold, "k": 5 if search_type == 'questions' else 10}
+        )
+
+        prompt_template = PromptTemplate(
+            input_variables=["context", "input"],
+            template="""Use the provided context to answer the question. Be concise and factual.
+
+Context: {context}
+
+Question: {input}
+
+Answer:"""
+        )
+        # Create the document chain and then the retrieval chain
+        document_chain = create_stuff_documents_chain(llm, prompt_template)
+        qa_chain = create_retrieval_chain(retriever, document_chain)
+
+    if search_type == 'items':
+        return _process_checklist_items(queries, vector_store, threshold, store_name, session)
+    else:
+        return _process_questions(queries, vector_store, threshold, qa_chain, llm)
+
+
+def _process_checklist_items(checklist: Dict, vector_store: FAISS, threshold: float, store_name: str = None, session=None) -> Dict:
+    """Compare checklist items directly against LLM-generated document type classifications"""
+
+    # Ensure checklist embeddings are preloaded first
+    if not hasattr(get_checklist_embedding, '_cache') or not get_checklist_embedding._cache:
+        logger.info("Checklist embeddings cache is empty, preloading...")
+        try:
+            from app.core.search import preload_checklist_embeddings
+            count = preload_checklist_embeddings()
+            logger.info(f"✅ Preloaded {count} checklist embeddings for processing")
+        except Exception as e:
+            logger.error(f"Failed to preload checklist embeddings: {e}")
+            return {}
+
+    # Ensure document type embeddings are available
+    if session:
+        logger.debug(f"Checklist processing session ID: {id(session)}, has embeddings: {hasattr(session, 'document_type_embeddings')}")
+        if hasattr(session, 'document_type_embeddings'):
+            logger.debug(f"Embeddings count: {len(session.document_type_embeddings) if session.document_type_embeddings else 0}")
+
+    # Try to auto-preload embeddings if missing
+    embeddings_missing = not session or not hasattr(session, 'document_type_embeddings') or not session.document_type_embeddings
+
+    if embeddings_missing and store_name:
+        logger.info(f"Document type embeddings missing, attempting auto-preload for {store_name}...")
+        try:
+            from app.core.search import preload_document_type_embeddings
+            type_embeddings = preload_document_type_embeddings(store_name)
+            if not hasattr(session, 'document_type_embeddings') or session.document_type_embeddings is None:
+                session.document_type_embeddings = {}
+            session.document_type_embeddings.update(type_embeddings)
+            logger.info(f"✅ Auto-preloaded {len(type_embeddings)} document type embeddings")
+            embeddings_missing = False
+        except Exception as e:
+            logger.warning(f"Failed to auto-preload document type embeddings: {e}")
+
+    if embeddings_missing:
+        logger.error("Document type embeddings not available. Checklist processing requires preloaded embeddings.")
+        logger.error("Make sure data room processing completed successfully or embeddings can be auto-loaded.")
+        return {}
+
+    # Load document type classifications - these are our primary comparison targets
+    doc_types = {}
+    if store_name:
+        doc_types = _load_document_types(vector_store, store_name)
+
+    if not doc_types:
+        logger.warning(f"No document type classifications found for {store_name}")
+        return {}
+
+    results = {}
+    for cat_letter, category in checklist.items():
+        cat_results = {
+            'name': category['name'],
+            'items': [],
+            'total_items': len(category['items']),
+            'matched_items': 0
+        }
+
+        for item in category['items']:
+            checklist_item_text = item['text'].lower().strip()
+            matches = []
+
+            # Compare checklist item against each document's type classification
+            for doc_path, doc_type in doc_types.items():
+                if not doc_type or doc_type == 'not classified':
+                    continue
+
+                doc_type_lower = doc_type.lower().strip()
+
+                # Calculate semantic similarity between checklist item and document type
+                try:
+                    # Get checklist embedding from memory cache (preloaded during data room processing)
+                    checklist_embedding = get_checklist_embedding(checklist_item_text)
+
+                    # Get document type embedding (from preloaded cache)
+                    doc_type_embedding = get_document_type_embedding(doc_type_lower, session)
+
+                    # Calculate cosine similarity
+                    import numpy as np
+                    similarity = np.dot(checklist_embedding, doc_type_embedding) / (
+                        np.linalg.norm(checklist_embedding) * np.linalg.norm(doc_type_embedding)
+                    )
+
+                    # Only include matches above threshold
+                    if similarity >= threshold:
+                        # Find the document metadata from the vector store
+                        # We need to get the document name and other metadata
+                        doc_name = _extract_doc_name_from_path(doc_path)
+
+                        matches.append({
+                            'name': doc_name,
+                            'path': doc_path,
+                            'full_path': doc_path,  # For consistency
+                            'score': round(float(similarity), 3),
+                            'document_type': doc_type,
+                            'text': f"Document type: {doc_type}"  # Include document type as text
+                        })
+
+                except Exception as e:
+                    logger.warning(f"Error calculating similarity for {doc_path}: {e}")
+                    continue
+
+            # Sort matches by score (highest first)
+            matches.sort(key=lambda x: x['score'], reverse=True)
+
+            # Limit to top matches for performance
+            matches = matches[:10]
+
+            if matches:
+                cat_results['matched_items'] += 1
+                logger.info(f"✅ Found {len(matches)} matches for checklist item: '{checklist_item_text[:50]}...'")
+
+            cat_results['items'].append({
+                'text': item['text'],
+                'original': item['original'],
+                'matches': matches
+            })
+
+        results[cat_letter] = cat_results
+
+    return results
+
+
+def _load_document_types(vector_store, store_name: str):
+    """Load document type classifications for the given store"""
+    try:
+        from pathlib import Path
+        from app.core.config import get_app_config
+        config = get_app_config()
+        doc_types_path = config.paths['faiss_dir'] / f"{store_name}_document_types.json"
+        if doc_types_path.exists():
+            import json
+            with open(doc_types_path, 'r') as f:
+                return json.load(f)
+    except Exception as e:
+        logger.warning(f"Failed to load document types for {store_name}: {e}")
+    return {}
+
+
+def _extract_doc_name_from_path(doc_path: str) -> str:
+    """Extract document name from file path"""
+    try:
+        path_obj = Path(doc_path)
+        return path_obj.name
+    except Exception:
+        # Fallback: extract name from path string
+        return doc_path.split('/')[-1] if '/' in doc_path else doc_path.split('\\')[-1] if '\\' in doc_path else doc_path
+
+
+def get_checklist_embedding(checklist_text: str):
+    """
+    Get cached embedding for checklist item from in-memory cache.
+
+    This function only uses in-memory cache that should be preloaded during
+    data room processing. It will fail if the embedding is not available.
+
+    Args:
+        checklist_text: The checklist item text to look up
+
+    Returns:
+        numpy array: The embedding vector for the checklist text
+
+    Raises:
+        RuntimeError: If embedding is not found in cache
+    """
+    # Initialize cache if not exists
+    if not hasattr(get_checklist_embedding, '_cache'):
+        get_checklist_embedding._cache = {}
+        logger.warning("Checklist embedding cache was not initialized - this should not happen!")
+
+    # Create cache key from checklist text with normalized Unicode
+    cache_key = checklist_text.lower().strip()
+    # Use unidecode for comprehensive Unicode to ASCII conversion
+    cache_key = unidecode.unidecode(cache_key)
+
+    # Check in-memory cache only
+    if cache_key in get_checklist_embedding._cache:
+        return get_checklist_embedding._cache[cache_key]
+
+    # Enhanced debugging for troubleshooting
+    cache_size = len(get_checklist_embedding._cache)
+    logger.warning(f"Checklist embedding not found: '{checklist_text[:50]}...'")
+    logger.warning(f"Cache key generated: '{cache_key}'")
+    logger.warning(f"Cache has {cache_size} items total")
+
+    if cache_size > 0:
+        # Look for similar keys to help debug
+        similar_keys = []
+        search_terms = checklist_text.lower().split()
+        for key in get_checklist_embedding._cache.keys():
+            if any(term in key for term in search_terms if len(term) > 3):
+                similar_keys.append(key)
+
+        if similar_keys:
+            logger.warning(f"Similar keys found: {similar_keys[:3]}")
+        else:
+            logger.warning("No similar keys found in cache")
+
+        # Show a few sample keys
+        sample_keys = list(get_checklist_embedding._cache.keys())[:5]
+        logger.warning(f"Sample cache keys: {sample_keys}")
+    else:
+        logger.error("Cache is completely empty - embeddings were not preloaded!")
+
+    # Fail if not found - no fallbacks
+    raise RuntimeError(
+        f"Checklist embedding not found for: '{checklist_text[:50]}...' (cache key: '{cache_key}'). "
+        f"Cache has {cache_size} items. "
+        "Make sure embeddings were preloaded during data room processing."
+    )
+
+
+def get_document_type_embedding(doc_type: str, session=None):
+    """
+    Get cached embedding for document type from session cache.
+
+    Args:
+        doc_type: The document type text to get embedding for
+        session: The session object containing preloaded embeddings
+
+    Returns:
+        numpy.ndarray: The embedding vector
+
+    Raises:
+        RuntimeError: If embedding is not found in cache
+    """
+    if not session or not hasattr(session, 'document_type_embeddings') or not session.document_type_embeddings:
+        raise RuntimeError(f"Document type embedding not found for: '{doc_type[:50]}...'. Preloaded embeddings required.")
+
+    # Create cache key with normalized Unicode
+    cache_key = unidecode.unidecode(doc_type.lower().strip())
+
+    # Get from session cache only
+    if cache_key in session.document_type_embeddings:
+        return session.document_type_embeddings[cache_key]
+
+    raise RuntimeError(f"Document type embedding not found for: '{doc_type[:50]}...' (cache key: '{cache_key}')")
+
+
+def generate_checklist_embeddings():
+    """
+    Generate embeddings for all checklist items and save to disk.
+
+    This function should be called during the build process to pre-calculate
+    embeddings for all checklist items from the available checklist files.
+
+    Returns:
+        int: Number of embeddings generated and saved
+    """
+    try:
+        from app.core.config import get_config
+        from app.core.model_cache import get_cached_embeddings
+        import json
+        import numpy as np
+
+        config = get_config()
+        embeddings_model = get_cached_embeddings()
+        checklist_dir = config.paths['checklist_dir']
+
+        logger.info("🔄 Generating checklist embeddings...")
+
+        # Initialize embeddings cache
+        embeddings_cache = {}
+
+        # Process all checklist files
+        checklist_files = list(checklist_dir.glob("*.md"))
+        if not checklist_files:
+            logger.warning(f"No checklist files found in {checklist_dir}")
+            return 0
+
+        for checklist_file in checklist_files:
+            logger.info(f"Processing checklist: {checklist_file.name}")
+
+            try:
+                # Read checklist content
+                content = checklist_file.read_text(encoding='utf-8')
+
+                # Parse checklist items from markdown
+                checklist_items = _parse_checklist_items_from_markdown(content)
+
+                # Generate embeddings for each item
+                for item_text in checklist_items:
+                    # Normalize Unicode in cache key
+                    cache_key = item_text.lower().strip()
+                    cache_key = unidecode.unidecode(cache_key)
+
+                    # Skip if already processed
+                    if cache_key in embeddings_cache:
+                        continue
+
+                    try:
+                        # Generate embedding
+                        embedding = embeddings_model.embed_query(item_text)
+
+                        # Handle both list and numpy array cases
+                        if hasattr(embedding, 'tolist'):
+                            embeddings_cache[cache_key] = embedding.tolist()
+                        else:
+                            # Already a list
+                            embeddings_cache[cache_key] = embedding
+
+                        logger.debug(f"✅ Embedded: {item_text[:50]}...")
+
+                    except Exception as e:
+                        logger.warning(f"Failed to embed checklist item '{item_text[:50]}...': {e}")
+                        continue
+
+            except Exception as e:
+                logger.error(f"Failed to process checklist file {checklist_file}: {e}")
+                continue
+
+        # Save to disk
+        cache_file = config.paths['faiss_dir'] / "checklist_embeddings.json"
+        cache_file.parent.mkdir(parents=True, exist_ok=True)
+
+        with open(cache_file, 'w', encoding='utf-8') as f:
+            json.dump(embeddings_cache, f, indent=2, ensure_ascii=False)
+
+        logger.info(f"💾 Saved {len(embeddings_cache)} checklist embeddings to {cache_file}")
+        return len(embeddings_cache)
+
+    except Exception as e:
+        error_msg = f"Failed to generate checklist embeddings: {e}"
+        logger.error(error_msg)
+        raise RuntimeError(error_msg)
+
+
+def _parse_checklist_items_from_markdown(content: str) -> list:
+    """
+    Parse checklist items from markdown content.
+
+    Args:
+        content: Markdown content containing checklist items
+
+    Returns:
+        list: List of checklist item texts
+    """
+    import re
+
+    items = []
+
+    # Find numbered items like "1. Item text" or "- Item text"
+    # Look for patterns after category headers
+    lines = content.split('\n')
+
+    for line in lines:
+        line = line.strip()
+
+        # Skip empty lines and headers
+        if not line or line.startswith('#') or line.startswith('⸻'):
+            continue
+
+        # Look for numbered items: "1. ", "2. ", etc. or bullet points
+        if re.match(r'^\d+\.\s+', line) or line.startswith('- '):
+            # Clean up the item text
+            if line.startswith('- '):
+                item_text = line[2:].strip()
+            else:
+                # Remove the number prefix
+                item_text = re.sub(r'^\d+\.\s+', '', line).strip()
+
+            # Skip if too short or looks like a header
+            if len(item_text) > 10 and not item_text.isupper():
+                items.append(item_text)
+
+    logger.info(f"Parsed {len(items)} checklist items from markdown")
+    return items
+
+
+def preload_checklist_embeddings():
+    """
+    Preload all checklist embeddings into memory during data room processing.
+
+    This function loads pre-calculated embeddings from disk into the in-memory cache.
+    It should be called once during data room processing to prepare for fast searches.
+
+    Returns:
+        int: Number of embeddings successfully preloaded
+
+    Raises:
+        RuntimeError: If embeddings file doesn't exist or can't be loaded
+    """
+    try:
+        from app.core.config import get_config
+        import json
+        import numpy as np
+
+        config = get_config()
+        cache_file = config.paths['faiss_dir'] / "checklist_embeddings.json"
+
+        if not cache_file.exists():
+            logger.warning(f"Checklist embeddings file not found: {cache_file}")
+            logger.info("Generating checklist embeddings now...")
+
+            # Try to generate embeddings on-the-fly
+            try:
+                generated_count = generate_checklist_embeddings()
+                if generated_count > 0:
+                    logger.info(f"✅ Generated {generated_count} embeddings, now preloading...")
+                else:
+                    raise RuntimeError("No checklist items found to embed")
+            except Exception as gen_error:
+                raise RuntimeError(
+                    f"Could not generate checklist embeddings: {gen_error}. "
+                    "Make sure checklist files exist and are properly formatted."
+                )
+
+        # Initialize cache
+        if not hasattr(get_checklist_embedding, '_cache'):
+            get_checklist_embedding._cache = {}
+
+        # Load all embeddings from disk
+        with open(cache_file, 'r', encoding='utf-8') as f:
+            cache_data = json.load(f)
+
+        # Convert and cache all embeddings in memory
+        preloaded_count = 0
+        for cache_key, embedding_list in cache_data.items():
+            # Normalize Unicode in cache key to match search normalization
+            normalized_key = unidecode.unidecode(cache_key)
+            embedding_array = np.array(embedding_list, dtype=np.float32)
+            get_checklist_embedding._cache[normalized_key] = embedding_array
+            preloaded_count += 1
+
+        logger.info(f"✅ Preloaded {preloaded_count} checklist embeddings into memory")
+        return preloaded_count
+
+    except Exception as e:
+        error_msg = f"Failed to preload checklist embeddings: {e}"
+        logger.error(error_msg)
+        raise RuntimeError(error_msg)
+
+
+def preload_document_type_embeddings(store_name: str):
+    """
+    Preload all document type embeddings into memory during data room processing.
+
+    This function loads document type classifications and computes their embeddings
+    once during data room processing to avoid runtime computation.
+
+    Returns:
+        dict: Dictionary mapping normalized document types to their embeddings
+
+    Raises:
+        RuntimeError: If document types can't be loaded or embeddings can't be computed
+    """
+    try:
+        from app.core.model_cache import get_cached_embeddings
+        import numpy as np
+
+        # Load document type classifications
+        doc_types = _load_document_types(None, store_name)
+        if not doc_types:
+            raise RuntimeError(f"No document type classifications found for {store_name}")
+
+        # Get embeddings model
+        embeddings = get_cached_embeddings()
+
+        # Precompute embeddings for all unique document types
+        type_embeddings = {}
+        unique_types = set()
+
+        # Collect all unique document types
+        for doc_path, doc_type in doc_types.items():
+            if doc_type and doc_type != 'not classified':
+                normalized_type = unidecode.unidecode(doc_type.lower().strip())
+                unique_types.add(normalized_type)
+
+        # Precompute embeddings for each unique type
+        for doc_type in unique_types:
+            try:
+                embedding = embeddings.embed_query(doc_type)
+                # Ensure it's a numpy array
+                if hasattr(embedding, 'tolist'):
+                    embedding_array = np.array(embedding, dtype=np.float32)
+                else:
+                    embedding_array = np.array(embedding, dtype=np.float32)
+                type_embeddings[doc_type] = embedding_array
+            except Exception as e:
+                logger.warning(f"Failed to compute embedding for document type '{doc_type}': {e}")
+                continue
+
+        logger.info(f"✅ Precomputed {len(type_embeddings)} document type embeddings")
+        return type_embeddings
+
+    except Exception as e:
+        error_msg = f"Failed to preload document type embeddings: {e}"
+        logger.error(error_msg)
+        raise RuntimeError(error_msg)
+
+
+
+
+def _process_questions(queries: List[Dict], vector_store: FAISS, threshold: float, qa_chain=None, llm=None) -> Dict:
+    """Process questions using batch processing for parallel LLM calls"""
+    if not queries:
+        return {'questions': []}
+    
+    if qa_chain and llm:
+        return _process_questions_with_rag_batch(queries, vector_store, threshold, llm)
+    elif qa_chain:
+        raise ValueError("LLM required for RAG processing but not provided")
+    else:
+        return _process_questions_simple_search(queries, vector_store, threshold)
+
+
+def _process_questions_with_rag_batch(queries: List[Dict], vector_store: FAISS, threshold: float, llm) -> Dict:
+    """Process questions using batch processing - fail fast, no fallbacks"""
+    from app.ai.agent_utils import create_batch_processor
+    from langchain_core.messages import HumanMessage
+    
+    # Create batch processor
+    batch_processor = create_batch_processor(llm, max_concurrency=5)
+    
+    logger.info(f"Processing {len(queries)} questions using batch processing")
+    
+    # Prepare all batch inputs
+    batch_inputs = []
+    question_contexts = []
+    
+    for query in queries:
+        question = query['question']
+        
+        # Retrieve documents for this question
+        docs_with_scores = vector_store.similarity_search_with_score(question, k=5)
+        relevant_docs = [doc for doc, score in docs_with_scores if (1.0 - (score / 2.0) if score <= 2.0 else 0.0) >= threshold]
+        
+        # Create context and sources
+        if relevant_docs:
+            context = "\n".join([f"- {doc.metadata.get('name', 'Unknown')}: {doc.page_content[:200]}..." 
+                               for doc in relevant_docs[:5]])
+            sources = [{'name': doc.metadata.get('name', ''), 
+                       'path': doc.metadata.get('path', ''),
+                       'score': round(1.0 - (score / 2.0) if score <= 2.0 else 0.0, 3)}
+                      for doc, score in docs_with_scores[:5] if (1.0 - (score / 2.0) if score <= 2.0 else 0.0) >= threshold]
+        else:
+            context = ""
+            sources = []
+        
+        question_contexts.append(sources)
+        
+        # Create prompt
+        prompt_content = f"""Use the provided context to answer the question. Be concise and factual.
+
+Context: {context}
+
+Question: {question}
+
+Answer:"""
+        
+        messages = [HumanMessage(content=prompt_content)]
+        batch_inputs.append((messages, query))
+    
+    # Process batch - fail if anything goes wrong
+    batch_results = batch_processor.invoke(batch_inputs)
+    
+    # Build results
+    results = []
+    for idx, result in enumerate(batch_results):
+        if not result['success'] or not result['response']:
+            raise RuntimeError(f"Failed to process question: {result['item_info']['question']}")
+        
+        query = result['item_info']
+        answer = result['response'].content.strip()
+        sources = question_contexts[idx]
+        
+        results.append({
+            'question': query['question'],
+            'category': query.get('category', ''),
+            'answer': answer,
+            'sources': sources,
+            'method': 'rag_batch',
+            'has_answer': bool(answer and answer.strip())
+        })
+    
+    return {'questions': results}
+
+
+
+
+def _process_questions_simple_search(queries: List[Dict], vector_store: FAISS, threshold: float) -> Dict:
+    """Process questions using simple search without RAG (already fast, no batch needed)"""
+    results = []
+    
+    for query in queries:
+        question = query['question']
+        category = query.get('category', '')
+        
+        # Simple search without RAG
+        docs_with_scores = vector_store.similarity_search_with_score(question, k=5)
+        sources = []
+        for doc, score in docs_with_scores:
+            if score >= threshold:
+                sources.append({
+                    'name': doc.metadata.get('name', ''),
+                    'path': doc.metadata.get('path', ''),
+                    'score': round(score, 3)
+                })
+
+        answer = f"Based on the following documents: {', '.join([s['name'] for s in sources])}" if sources else "No relevant documents found"
+        results.append({
+            'question': question,
+            'category': category,
+            'answer': answer,
+            'sources': sources,
+            'method': 'search',
+            'has_answer': bool(sources)
+        })
+
+    return {'questions': results}
+
+
+def search_documents(query: str, document_processor: DocumentProcessor, top_k: int = 5, threshold: float = None):
+    """Search documents using the document processor"""
+    if not document_processor:
+        return []
+
+    return document_processor.search(query, top_k=top_k, threshold=threshold)
+
+
+def hybrid_search(query: str, vector_store: FAISS, store_name: str,
+                 top_k: int = 10, sparse_weight: float = 0.3,
+                 dense_weight: float = 0.7, threshold: float = SIMILARITY_THRESHOLD) -> List[Dict]:
+    """
+    Hybrid search combining sparse (BM25) and dense retrieval.
+
+    Args:
+        query: Search query
+        vector_store: FAISS vector store for dense retrieval
+        store_name: Name of the document store
+        top_k: Number of top results to return
+        sparse_weight: Weight for sparse scores (0-1)
+        dense_weight: Weight for dense scores (0-1)
+        threshold: Minimum similarity threshold for dense retrieval
+
+    Returns:
+        Combined search results sorted by hybrid score
+    """
+    logger.info(f"Performing hybrid search for query: {query[:50]}...")
+
+    # Get sparse results
+    sparse_results = []
+    bm25_index = load_sparse_index_for_store(store_name)
+
+    if bm25_index:
+        sparse_results = bm25_index.search(query, top_k=top_k*2)
+        logger.info(f"Sparse search returned {len(sparse_results)} results")
+    else:
+        logger.warning(f"No sparse index found for {store_name}, falling back to dense only")
+
+    # Get dense results
+    dense_docs = vector_store.similarity_search_with_score(query, k=top_k*2)
+    dense_results = []
+
+    for doc, score in dense_docs:
+        if score >= threshold:
+            dense_results.append({
+                'doc_id': doc.metadata.get('source', ''),
+                'document': doc.page_content,
+                'score': float(score),
+                'metadata': doc.metadata
+            })
+
+    logger.info(f"Dense search returned {len(dense_results)} results")
+
+    # Combine results using reciprocal rank fusion or weighted scoring
+    combined_scores = {}
+
+    # Process sparse results
+    for result in sparse_results:
+        doc_id = result['doc_id']
+        combined_scores[doc_id] = {
+            'sparse_score': result['score'] * sparse_weight,
+            'dense_score': 0.0,
+            'result': result
+        }
+
+    # Process dense results
+    for result in dense_results:
+        doc_id = result['doc_id']
+        if doc_id in combined_scores:
+            combined_scores[doc_id]['dense_score'] = result['score'] * dense_weight
+        else:
+            combined_scores[doc_id] = {
+                'sparse_score': 0.0,
+                'dense_score': result['score'] * dense_weight,
+                'result': result
+            }
+
+    # Calculate final hybrid scores
+    final_results = []
+    for doc_id, scores in combined_scores.items():
+        hybrid_score = scores['sparse_score'] + scores['dense_score']
+
+        # Create unified result format
+        result = scores['result'].copy()
+        result.update({
+            'hybrid_score': hybrid_score,
+            'sparse_score': scores['sparse_score'] / sparse_weight if sparse_weight > 0 else 0,
+            'dense_score': scores['dense_score'] / dense_weight if dense_weight > 0 else 0,
+            'score': hybrid_score  # For backward compatibility
+        })
+        final_results.append(result)
+
+    # Sort by hybrid score
+    final_results.sort(key=lambda x: x['hybrid_score'], reverse=True)
+
+    # Return top_k results
+    top_results = final_results[:top_k]
+    logger.info(f"Hybrid search returned {len(top_results)} final results")
+
+    return top_results
+
+

app/core/sparse_index.py

@@ -0,0 +1,263 @@
+#!/usr/bin/env python3
+"""
+BM25 Sparse Index Implementation for Due Diligence Documents
+
+This module provides BM25-based sparse retrieval that complements the existing
+dense retrieval system. The index is pre-calculated locally and persisted
+to disk for fast loading on Streamlit Cloud.
+"""
+
+import pickle
+import os
+import re
+from typing import List, Dict, Optional, Callable, Tuple
+from pathlib import Path
+
+from rank_bm25 import BM25Okapi
+from app.core.logging import logger
+
+
+class BM25Index:
+    """
+    BM25-based sparse index for document retrieval.
+
+    This class provides:
+    - Pre-calculated BM25 index persistence
+    - Custom tokenization for legal/financial documents
+    - Efficient search with relevance scoring
+    - Integration with existing document processing pipeline
+    """
+
+    def __init__(self, index_path: str):
+        """
+        Initialize BM25 index.
+
+        Args:
+            index_path: Path to save/load the index file
+        """
+        self.index_path = Path(index_path)
+        self.bm25: Optional[BM25Okapi] = None
+        self.documents: List[str] = []
+        self.doc_ids: List[str] = []
+        self.tokenized_docs: List[List[str]] = []
+        self.metadata: Dict = {}
+
+    def custom_tokenizer(self, text: str) -> List[str]:
+        """
+        Custom tokenization optimized for legal and financial documents.
+
+        Handles:
+        - Legal abbreviations (LLC, Inc., Corp.)
+        - Financial terms (IPO, GAAP, EBITDA)
+        - Contract terminology (force majeure, indemnification)
+        - Proper names and entities
+        """
+        if not text:
+            return []
+
+        # Convert to lowercase
+        text = text.lower()
+
+        # Preserve important legal/financial abbreviations
+        legal_abbrevs = [
+            'llc', 'inc', 'corp', 'ltd', 'co', 'lp', 'llp',
+            'ipo', 'gaap', 'sec', 'fdic', 'irs', 'sox', 'gdpr',
+            'nda', 'mou', 'spa', 'joa', 'ipa', 'dpa'
+        ]
+
+        # Replace common terms to avoid splitting
+        for abbrev in legal_abbrevs:
+            text = text.replace(abbrev, abbrev.replace(' ', '_'))
+
+        # Split on whitespace and punctuation
+        tokens = re.findall(r'\b\w+\b', text)
+
+        # Restore underscores to spaces for abbreviations
+        tokens = [token.replace('_', '') for token in tokens]
+
+        # Filter out very short tokens (likely noise)
+        tokens = [token for token in tokens if len(token) > 1]
+
+        return tokens
+
+    def build_index(self, documents: List[Dict[str, str]], custom_tokenizer: Optional[Callable] = None):
+        """
+        Build BM25 index from documents.
+
+        Args:
+            documents: List of dicts with 'id' and 'content' keys
+            custom_tokenizer: Optional custom tokenization function
+        """
+        logger.info(f"Building BM25 index from {len(documents)} documents")
+
+        # Extract content and IDs
+        self.documents = [doc['content'] for doc in documents]
+        self.doc_ids = [doc['id'] for doc in documents]
+
+        # Tokenize documents
+        tokenizer = custom_tokenizer or self.custom_tokenizer
+        self.tokenized_docs = [tokenizer(doc) for doc in self.documents]
+
+        # Build BM25 index
+        self.bm25 = BM25Okapi(self.tokenized_docs)
+
+        # Store metadata
+        self.metadata = {
+            'total_documents': len(self.documents),
+            'total_tokens': sum(len(tokens) for tokens in self.tokenized_docs),
+            'avg_tokens_per_doc': sum(len(tokens) for tokens in self.tokenized_docs) / len(self.documents) if self.documents else 0
+        }
+
+        # Save to disk
+        self._save_index()
+
+        logger.info(f"✅ BM25 index built and saved: {self.metadata}")
+
+    def _save_index(self):
+        """Save index to pickle file"""
+        self.index_path.parent.mkdir(parents=True, exist_ok=True)
+
+        index_data = {
+            'bm25': self.bm25,
+            'documents': self.documents,
+            'doc_ids': self.doc_ids,
+            'tokenized_docs': self.tokenized_docs,
+            'metadata': self.metadata
+        }
+
+        with open(self.index_path, 'wb') as f:
+            pickle.dump(index_data, f)
+
+        logger.info(f"💾 BM25 index saved to {self.index_path}")
+
+    def load_index(self) -> bool:
+        """
+        Load index from disk.
+
+        Returns:
+            True if index loaded successfully, False otherwise
+        """
+        if self.index_path.exists():
+            try:
+                with open(self.index_path, 'rb') as f:
+                    index_data = pickle.load(f)
+
+                self.bm25 = index_data['bm25']
+                self.documents = index_data['documents']
+                self.doc_ids = index_data['doc_ids']
+                self.tokenized_docs = index_data['tokenized_docs']
+                self.metadata = index_data.get('metadata', {})
+
+                logger.info(f"📂 BM25 index loaded: {len(self.documents)} documents")
+                return True
+
+            except Exception as e:
+                logger.error(f"Failed to load BM25 index: {e}")
+                return False
+        else:
+            logger.warning(f"BM25 index not found: {self.index_path}")
+            return False
+
+    def search(self, query: str, top_k: int = 10, custom_tokenizer: Optional[Callable] = None) -> List[Dict]:
+        """
+        Search the BM25 index.
+
+        Args:
+            query: Search query
+            top_k: Number of top results to return
+            custom_tokenizer: Optional custom tokenization function
+
+        Returns:
+            List of search results with scores
+        """
+        if not self.bm25:
+            logger.warning("BM25 index not loaded")
+            return []
+
+        # Tokenize query
+        tokenizer = custom_tokenizer or self.custom_tokenizer
+        tokenized_query = tokenizer(query)
+
+        if not tokenized_query:
+            logger.warning("Query produced no tokens")
+            return []
+
+        # Get BM25 scores
+        scores = self.bm25.get_scores(tokenized_query)
+
+        # Get top results
+        if len(scores) == 0:
+            return []
+
+        # Get indices of top scores (handling edge case of fewer results than requested)
+        num_results = min(top_k, len(scores))
+        top_indices = scores.argsort()[-num_results:][::-1]
+
+        results = []
+        for idx in top_indices:
+            if scores[idx] > 0:  # Only return relevant results
+                results.append({
+                    'doc_id': self.doc_ids[idx],
+                    'document': self.documents[idx],
+                    'score': float(scores[idx]),
+                    'rank': len(results) + 1
+                })
+
+        logger.debug(f"BM25 search returned {len(results)} results for query: {query[:50]}...")
+        return results
+
+    def get_stats(self) -> Dict:
+        """Get index statistics"""
+        if not self.index_path.exists():
+            return {'status': 'index_not_found'}
+
+        stats = {
+            'index_path': str(self.index_path),
+            'index_exists': self.index_path.exists(),
+            'is_loaded': self.bm25 is not None,
+            'index_size_mb': self.index_path.stat().st_size / (1024 * 1024) if self.index_path.exists() else 0
+        }
+
+        if self.metadata:
+            stats.update(self.metadata)
+
+        return stats
+
+
+def build_sparse_index_for_store(store_name: str, documents: List[Dict[str, str]],
+                                index_dir: str = "data/search_indexes") -> BM25Index:
+    """
+    Convenience function to build sparse index for a document store.
+
+    Args:
+        store_name: Name of the document store (e.g., 'summit-digital-solutions-inc')
+        documents: List of documents with 'id' and 'content' keys
+        index_dir: Directory to store the index
+
+    Returns:
+        BM25Index instance
+    """
+    index_path = f"{index_dir}/{store_name}_bm25.pkl"
+    bm25_index = BM25Index(index_path)
+    bm25_index.build_index(documents)
+    return bm25_index
+
+
+def load_sparse_index_for_store(store_name: str, index_dir: str = "data/search_indexes") -> Optional[BM25Index]:
+    """
+    Convenience function to load sparse index for a document store.
+
+    Args:
+        store_name: Name of the document store
+        index_dir: Directory containing the index
+
+    Returns:
+        BM25Index instance if found, None otherwise
+    """
+    index_path = f"{index_dir}/{store_name}_bm25.pkl"
+    bm25_index = BM25Index(index_path)
+
+    if bm25_index.load_index():
+        return bm25_index
+
+    return None

app/core/stage_manager.py

@@ -0,0 +1,326 @@
+#!/usr/bin/env python3
+"""
+Stage-based Build System for FAISS Index Generation
+
+This module provides a stage-based build system that allows for incremental
+builds, dependency management, and smart skipping of completed stages.
+"""
+
+import json
+import logging
+import time
+from pathlib import Path
+from typing import Dict, List, Any, Optional, Set
+from datetime import datetime
+import glob
+
+logger = logging.getLogger(__name__)
+
+# Stage definitions with dependencies and outputs
+STAGES = {
+    'scan': {
+        'name': 'Document Scanning',
+        'description': 'Scan and catalog all documents',
+        'dependencies': [],
+        'outputs': ['.scan_cache.json'],
+        'estimated_duration': '30s'
+    },
+    'extract': {
+        'name': 'Text Extraction',
+        'description': 'Extract text from PDFs and documents',
+        'dependencies': ['scan'],
+        'outputs': ['.extraction_cache.json'],
+        'estimated_duration': '5-10m'
+    },
+    'classify': {
+        'name': 'Document Classification',
+        'description': 'Classify document types using AI',
+        'dependencies': ['extract'],
+        'outputs': ['*_document_types.json'],
+        'estimated_duration': '3-5m'
+    },
+    'chunk': {
+        'name': 'Text Chunking',
+        'description': 'Split documents into semantic chunks',
+        'dependencies': ['extract'],
+        'outputs': ['.chunking_cache.json'],
+        'estimated_duration': '2-3m'
+    },
+    'embed': {
+        'name': 'Vector Embeddings',
+        'description': 'Generate embeddings for all chunks',
+        'dependencies': ['chunk'],
+        'outputs': ['*.pkl'],
+        'estimated_duration': '5-8m'
+    },
+    'index': {
+        'name': 'FAISS Indexing',
+        'description': 'Build and save FAISS vector indices',
+        'dependencies': ['embed'],
+        'outputs': ['*.faiss'],
+        'estimated_duration': '1-2m'
+    },
+    'sparse': {
+        'name': 'BM25 Sparse Indexing',
+        'description': 'Build BM25 sparse indices for hybrid search',
+        'dependencies': ['extract'],
+        'outputs': ['*_bm25.pkl'],
+        'estimated_duration': '2-3m'
+    }
+}
+
+
+class StageTracker:
+    """Tracks the state and completion status of build stages"""
+
+    def __init__(self, faiss_dir: Path):
+        self.faiss_dir = faiss_dir
+        self.state_file = faiss_dir / '.build_state.json'
+        self.state = self._load_state()
+
+    def _load_state(self) -> Dict[str, Any]:
+        """Load current build state from disk"""
+        if self.state_file.exists():
+            try:
+                return json.loads(self.state_file.read_text())
+            except json.JSONDecodeError as e:
+                logger.warning(f"Corrupted state file, starting fresh: {e}")
+                return self._create_initial_state()
+        else:
+            return self._create_initial_state()
+
+    def _create_initial_state(self) -> Dict[str, Any]:
+        """Create initial state structure"""
+        return {
+            'stages': {},
+            'last_build': None,
+            'version': '1.0',
+            'total_builds': 0
+        }
+
+    def _save_state(self):
+        """Save current state to disk"""
+        self.state_file.parent.mkdir(parents=True, exist_ok=True)
+        self.state_file.write_text(json.dumps(self.state, indent=2))
+
+    def is_stage_complete(self, stage_name: str) -> bool:
+        """Check if stage is complete and all outputs exist"""
+        if stage_name not in self.state['stages']:
+            return False
+
+        stage_info = self.state['stages'][stage_name]
+        stage_config = STAGES[stage_name]
+
+        # Check if all output files exist
+        for output_pattern in stage_config['outputs']:
+            pattern_path = self.faiss_dir / output_pattern
+            if not glob.glob(str(pattern_path)):
+                logger.debug(f"Missing output: {pattern_path}")
+                return False
+
+        return True
+
+    def mark_stage_complete(self, stage_name: str, metadata: dict = None):
+        """Mark stage as completed with metadata"""
+        self.state['stages'][stage_name] = {
+            'completed_at': datetime.now().isoformat(),
+            'metadata': metadata or {}
+        }
+        self._save_state()
+
+    def mark_stage_failed(self, stage_name: str, error: str):
+        """Mark stage as failed"""
+        self.state['stages'][stage_name] = {
+            'failed_at': datetime.now().isoformat(),
+            'error': error,
+            'status': 'failed'
+        }
+        self._save_state()
+
+    def should_skip_stage(self, stage_name: str, force_clean: bool) -> bool:
+        """Determine if stage should be skipped"""
+        if force_clean:
+            return False
+        return self.is_stage_complete(stage_name)
+
+    def get_stage_status(self, stage_name: str) -> Dict[str, Any]:
+        """Get detailed status of a stage"""
+        if stage_name not in self.state['stages']:
+            return {'status': 'not_started'}
+
+        stage_info = self.state['stages'][stage_name]
+        is_complete = self.is_stage_complete(stage_name)
+
+        return {
+            'status': 'completed' if is_complete else 'incomplete',
+            'completed_at': stage_info.get('completed_at'),
+            'metadata': stage_info.get('metadata', {}),
+            'error': stage_info.get('error'),
+            'is_complete': is_complete
+        }
+
+    def get_build_summary(self) -> Dict[str, Any]:
+        """Get summary of current build state"""
+        completed_stages = []
+        incomplete_stages = []
+        failed_stages = []
+
+        for stage_name in STAGES.keys():
+            status = self.get_stage_status(stage_name)
+            if status['status'] == 'completed':
+                completed_stages.append(stage_name)
+            elif status.get('error'):
+                failed_stages.append(stage_name)
+            else:
+                incomplete_stages.append(stage_name)
+
+        return {
+            'completed_stages': completed_stages,
+            'incomplete_stages': incomplete_stages,
+            'failed_stages': failed_stages,
+            'last_build': self.state.get('last_build'),
+            'total_builds': self.state.get('total_builds', 0)
+        }
+
+    def reset_stage(self, stage_name: str):
+        """Reset a specific stage to not started"""
+        if stage_name in self.state['stages']:
+            del self.state['stages'][stage_name]
+            self._save_state()
+
+    def reset_all_stages(self):
+        """Reset all stages to not started"""
+        self.state['stages'] = {}
+        self._save_state()
+
+
+class StageManager:
+    """Manages execution of build stages with dependency resolution"""
+
+    def __init__(self, faiss_dir: Path):
+        self.faiss_dir = faiss_dir
+        self.tracker = StageTracker(faiss_dir)
+
+    def resolve_dependencies(self, target_stages: List[str], completed_stages: Set[str]) -> List[str]:
+        """Resolve which stages need to run based on dependencies"""
+        to_run = []
+
+        for stage_name in target_stages:
+            if stage_name not in STAGES:
+                raise ValueError(f"Unknown stage: {stage_name}")
+
+            # Check dependencies recursively
+            for dep in STAGES[stage_name]['dependencies']:
+                if dep not in completed_stages:
+                    dep_chain = self.resolve_dependencies([dep], completed_stages)
+                    to_run.extend(dep_chain)
+
+            if stage_name not in completed_stages:
+                to_run.append(stage_name)
+
+        # Remove duplicates while preserving order
+        seen = set()
+        result = []
+        for stage in to_run:
+            if stage not in seen:
+                seen.add(stage)
+                result.append(stage)
+
+        return result
+
+    def get_completed_stages(self, force_clean: bool = False) -> Set[str]:
+        """Get set of completed stages"""
+        if force_clean:
+            return set()
+
+        completed = set()
+        for stage_name in STAGES.keys():
+            if self.tracker.is_stage_complete(stage_name):
+                completed.add(stage_name)
+        return completed
+
+    def execute_stage(self, stage_name: str, **kwargs) -> Dict[str, Any]:
+        """Execute a specific stage - to be implemented by subclasses"""
+        raise NotImplementedError(f"Stage execution not implemented for: {stage_name}")
+
+    def run_build_pipeline(self, target_stages: Optional[List[str]] = None,
+                          force_clean: bool = False) -> Dict[str, Any]:
+        """Run the build pipeline with dependency resolution"""
+
+        # Default to all stages if none specified
+        if target_stages is None:
+            target_stages = list(STAGES.keys())
+
+        # Get completed stages
+        completed_stages = self.get_completed_stages(force_clean)
+
+        # Resolve which stages need to run
+        stages_to_run = self.resolve_dependencies(target_stages, completed_stages)
+
+        logger.info(f"Build pipeline: {len(stages_to_run)} stages to execute")
+
+        results = []
+        for stage_name in stages_to_run:
+            stage_config = STAGES[stage_name]
+
+            if self.tracker.should_skip_stage(stage_name, force_clean):
+                logger.info(f"⏭️  Skipping stage '{stage_name}' (already complete)")
+                results.append({
+                    'stage': stage_name,
+                    'status': 'skipped',
+                    'reason': 'already_complete'
+                })
+                continue
+
+            logger.info(f"🚀 Executing stage '{stage_name}': {stage_config['description']}")
+            start_time = time.time()
+
+            try:
+                # Execute the stage
+                result = self.execute_stage(stage_name, force_clean=force_clean)
+
+                # Mark as complete
+                execution_time = time.time() - start_time
+                self.tracker.mark_stage_complete(stage_name, {
+                    'execution_time': execution_time,
+                    'result': result
+                })
+
+                logger.info(f"✅ Stage '{stage_name}' completed in {execution_time:.1f}s")
+                results.append({
+                    'stage': stage_name,
+                    'status': 'completed',
+                    'execution_time': execution_time,
+                    'result': result
+                })
+
+            except Exception as e:
+                execution_time = time.time() - start_time
+                error_msg = f"Stage '{stage_name}' failed after {execution_time:.1f}s: {e}"
+                logger.error(f"❌ {error_msg}")
+
+                self.tracker.mark_stage_failed(stage_name, str(e))
+
+                results.append({
+                    'stage': stage_name,
+                    'status': 'failed',
+                    'execution_time': execution_time,
+                    'error': str(e)
+                })
+
+                # Don't continue with dependent stages on failure
+                break
+
+        # Update build metadata
+        self.tracker.state['last_build'] = datetime.now().isoformat()
+        self.tracker.state['total_builds'] = self.tracker.state.get('total_builds', 0) + 1
+        self.tracker._save_state()
+
+        return {
+            'success': all(r['status'] in ['completed', 'skipped'] for r in results),
+            'stages_executed': len([r for r in results if r['status'] == 'completed']),
+            'stages_skipped': len([r for r in results if r['status'] == 'skipped']),
+            'stages_failed': len([r for r in results if r['status'] == 'failed']),
+            'results': results,
+            'total_time': sum(r.get('execution_time', 0) for r in results)
+        }

app/core/utils.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+"""
+Utility Functions Module
+
+Collection of utility functions used throughout the application.
+This module contains helper functions for file operations, formatting,
+and document processing utilities.
+"""
+
+from typing import List, Optional
+from pathlib import Path
+
+
+def get_mime_type(file_path: Path) -> str:
+    """Get MIME type based on file extension"""
+    file_extension = file_path.suffix.lower()
+    if file_extension == '.pdf':
+        return 'application/pdf'
+    elif file_extension in ['.doc', '.docx']:
+        return 'application/msword'
+    elif file_extension == '.txt':
+        return 'text/plain'
+    elif file_extension == '.md':
+        return 'text/markdown'
+    else:
+        return 'application/octet-stream'
+
+
+def format_document_title(doc_name: str) -> str:
+    """Format document name into a readable title"""
+    if '.' in doc_name:
+        doc_title = doc_name.rsplit('.', 1)[0].replace('_', ' ').replace('-', ' ').title()
+    else:
+        doc_title = doc_name.replace('_', ' ').replace('-', ' ').title()
+    return doc_title
+
+
+def count_documents_in_directory(directory: Path, supported_extensions: Optional[List[str]] = None) -> int:
+    """Count supported documents in a directory recursively"""
+    if supported_extensions is None:
+        supported_extensions = ['.pdf', '.docx', '.doc', '.txt', '.md']
+
+    return sum(1 for f in directory.rglob('*')
+               if f.is_file() and f.suffix.lower() in supported_extensions)
+
+
+def create_document_processor(store_name: Optional[str] = None) -> 'DocumentProcessor':
+    """
+    Create and initialize a DocumentProcessor.
+
+    This utility function encapsulates the common pattern of creating a DocumentProcessor
+    instance.
+
+    Args:
+        store_name: Optional name for the FAISS store (uses config default if None)
+
+    Returns:
+        Initialized DocumentProcessor instance
+    """
+    from app.core.document_processor import DocumentProcessor
+
+    # Create document processor instance
+    processor = DocumentProcessor(store_name=store_name)
+
+    return processor

app/handlers/__init__.py

@@ -0,0 +1,11 @@
+"""
+Handlers Package
+
+Contains business logic handlers that coordinate between UI and services.
+"""
+
+from .document_handler import DocumentHandler
+from .ai_handler import AIHandler
+from .export_handler import ExportHandler
+
+__all__ = ['DocumentHandler', 'AIHandler', 'ExportHandler']

app/handlers/ai_handler.py

@@ -0,0 +1,180 @@
+#!/usr/bin/env python3
+"""
+AI Handler
+
+Handles AI operations and coordinates between UI and AI service.
+"""
+
+from typing import Optional, List
+
+from app.ui.session_manager import SessionManager
+from app.services.ai_service import AIService, create_ai_service
+from app.core.exceptions import AIError, ConfigError, create_ai_error
+from app.ui.error_handler import handle_processing_errors
+from app.core.logging import logger
+
+
+class AIHandler:
+    """
+    AI handler that manages AI operations using the AI service.
+
+    Provides a clean interface between UI and AI service.
+    """
+
+    def __init__(self, session: SessionManager):
+        """Initialize handler with session manager"""
+        self.session = session
+        self._ai_service: Optional[AIService] = None
+
+    @handle_processing_errors("AI service setup", "Please check your API key and try again")
+    def setup_agent(self, api_key: str, model_choice: str) -> bool:
+        """
+        Setup AI service with given credentials.
+
+        Args:
+            api_key: Anthropic API key
+            model_choice: Claude model to use
+
+        Returns:
+            True if AI service was successfully initialized
+
+        Raises:
+            AIError: If AI service setup fails
+            ConfigError: If API key or model is invalid
+        """
+        # Get appropriate max_tokens for the model
+        from app.core.config import get_app_config
+        config = get_app_config()
+        
+        # Adjust max_tokens based on model limitations
+        max_tokens = config.model['max_tokens']
+        original_max_tokens = max_tokens
+        
+        if 'haiku' in model_choice.lower():
+            # Claude Haiku has a maximum of 8192 output tokens
+            max_tokens = min(max_tokens, 8192)
+        elif 'sonnet' in model_choice.lower():
+            # Claude Sonnet models can handle higher token counts
+            max_tokens = min(max_tokens, 8192)  # Conservative limit for reliability
+        
+        if max_tokens != original_max_tokens:
+            logger.info(f"Adjusted max_tokens for {model_choice}: {original_max_tokens} -> {max_tokens}")
+        
+        logger.info(f"Initializing AI service: model={model_choice}, max_tokens={max_tokens}, temperature={config.model['temperature']}")
+        
+        # Create AI service with proper token limits
+        self._ai_service = create_ai_service(
+            api_key=api_key, 
+            model=model_choice,
+            temperature=config.model['temperature'],
+            max_tokens=max_tokens
+        )
+
+        # Check if service was created successfully
+        if self._ai_service is None:
+            raise create_ai_error(
+                "AI service creation failed",
+                recovery_hint="Please check your API key and try again"
+            )
+
+        # Test the service
+        if self._ai_service.is_available:
+            # Store the AI service in the session for other components to access
+            self.session.agent = self._ai_service
+            return True
+        else:
+            raise create_ai_error(
+                "AI service initialization failed",
+                recovery_hint="Please check your API key and network connection"
+            )
+
+    def is_agent_available(self) -> bool:
+        """
+        Check if AI service is available and ready.
+
+        Returns:
+            True if AI service is available
+        """
+        # Check local AI service first
+        if self._ai_service is not None and self._ai_service.is_available:
+            return True
+        
+        # Check session for existing agent
+        if self.session.agent is not None:
+            # Update local reference if session has an agent
+            self._ai_service = self.session.agent
+            return self._ai_service.is_available
+        
+        return False
+
+
+    @handle_processing_errors("Report generation", "Please check your documents and try again")
+    def generate_report(self, report_type: str, **kwargs) -> Optional[str]:
+        """
+        Generate a report using the AI service.
+
+        Args:
+            report_type: Type of report ('overview', 'strategic', 'checklist', 'questions')
+            **kwargs: Additional arguments for report generation
+
+        Returns:
+            Generated report content or None if failed
+
+        Raises:
+            AIError: If report generation fails
+        """
+        if not self.is_agent_available():
+            raise create_ai_error(
+                "AI service not available",
+                recovery_hint="Please configure your API key in the sidebar"
+            )
+
+        documents = kwargs.get('documents', {})
+        strategy_text = kwargs.get('strategy_text')
+        checklist_results = kwargs.get('checklist_results')
+
+        return self._ai_service.analyze_documents(
+            documents=documents,
+            analysis_type=report_type,
+            strategy_text=strategy_text,
+            checklist_results=checklist_results
+        )
+
+
+    @handle_processing_errors("Question answering", "Please try rephrasing your question")
+    def answer_question(self, question: str, context_docs: List[str]) -> str:
+        """
+        Answer a specific question using AI.
+
+        Args:
+            question: The question to answer
+            context_docs: List of relevant document excerpts
+
+        Returns:
+            AI-generated answer
+
+        Raises:
+            AIError: If question answering fails
+        """
+        if not self.is_agent_available():
+            raise create_ai_error(
+                "AI service not available",
+                recovery_hint="Please configure your API key in the sidebar"
+            )
+
+        return self._ai_service.answer_question(question, context_docs)
+
+    @property
+    def llm(self):
+        """Get the underlying LLM instance"""
+        # Check local AI service first
+        if self._ai_service is not None:
+            return self._ai_service.llm
+        
+        # Check session for existing agent
+        if self.session.agent is not None:
+            # Update local reference if session has an agent
+            self._ai_service = self.session.agent
+            return self._ai_service.llm
+        
+        return None

app/handlers/document_handler.py

@@ -0,0 +1,230 @@
+#!/usr/bin/env python3
+"""
+Document Handler
+
+Handles document processing operations and coordinates with the document processor.
+"""
+
+from pathlib import Path
+from typing import Dict, List, Any
+
+from app.ui.session_manager import SessionManager
+from app.core.exceptions import ProcessingError
+from app.ui.error_handler import ErrorHandler, handle_processing_errors
+from app.core.exceptions import DocumentProcessingError, FileOperationError, create_processing_error
+from app.core.logging import logger
+
+
+class DocumentHandler:
+    """
+    Document handler that manages document processing operations.
+    """
+
+    def __init__(self, session: SessionManager):
+        """Initialize handler with session manager"""
+        self.session = session
+
+    @handle_processing_errors("Data room processing", "Please check that the data room exists and contains documents")
+    def process_data_room_fast(self, data_room_path: str):
+        """
+        Fast data room processing using pre-built FAISS indices.
+
+        Args:
+            data_room_path: Path to the data room directory
+
+        Returns:
+            Tuple of (documents_count, chunks_count) or None on error
+        """
+        # Extract company name from path
+        company_name = Path(data_room_path).name.lower()
+
+        # Initialize document processor with loaded FAISS store
+        from app.core.utils import create_document_processor
+        document_processor = create_document_processor(store_name=company_name)
+
+        if not document_processor.vector_store:
+            raise create_processing_error(
+                f"No pre-built FAISS index found for '{company_name}'",
+                recovery_hint="Please run scripts/build_indexes.py first to create the index"
+            )
+
+        # Quick document metadata scan
+        documents_dict = self._quick_document_scan(data_room_path)
+
+        # Get chunks from FAISS metadata
+        chunks = self._extract_chunks_from_faiss(document_processor)
+
+        # Store in session
+        self.session.documents = documents_dict
+        self.session.chunks = chunks
+        self.session.embeddings = document_processor.embeddings
+        self.session.vdr_store = company_name
+
+        # Preload checklist embeddings into memory for fast search
+        from app.core.search import preload_checklist_embeddings
+        logger.info("Attempting to preload checklist embeddings...")
+        try:
+            preloaded_count = preload_checklist_embeddings()
+            logger.info(f"✅ Successfully preloaded {preloaded_count} checklist embeddings for fast searching")
+        except RuntimeError as e:
+            logger.error(f"❌ Failed to preload checklist embeddings: {e}")
+            logger.error("This will cause checklist matching to fail - embeddings must be available for search")
+            # Don't fail the entire data room processing, but make it very clear this is a problem
+            raise  # Re-raise to make this a hard failure
+
+        # Preload document type embeddings into memory for fast search
+        from app.core.search import preload_document_type_embeddings
+        logger.info("Attempting to preload document type embeddings...")
+        try:
+            type_embeddings = preload_document_type_embeddings(company_name)
+            # Store in session for use during search
+            self.session.document_type_embeddings = type_embeddings
+            logger.info(f"✅ Successfully preloaded {len(type_embeddings)} document type embeddings for fast searching")
+            logger.info(f"Session ID: {id(self.session)}, Embeddings stored: {bool(self.session.document_type_embeddings)}")
+        except RuntimeError as e:
+            logger.error(f"❌ Failed to preload document type embeddings: {e}")
+            logger.error("Checklist processing will fail - embeddings are required")
+            raise  # Make this a hard failure since embeddings are now required
+
+        # Clear existing analysis
+        self.session.reset()
+
+        logger.info(f"Successfully processed {len(documents_dict)} documents and {len(chunks)} chunks")
+        return len(documents_dict), len(chunks)
+
+    def _quick_document_scan(self, data_room_path: str) -> Dict[str, Any]:
+        """Quick scan of document files without loading content"""
+        documents_dict = {}
+        data_room_path_obj = Path(data_room_path)
+
+        # Validate data room path exists
+        if not data_room_path_obj.exists():
+            raise create_processing_error(
+                f"Data room path does not exist: {data_room_path}",
+                recovery_hint="Please select a valid data room directory"
+            )
+
+        # Quick file system scan for supported extensions
+        from app.core import get_config
+        config = get_config()
+        supported_extensions = config.get_supported_extensions()
+
+        for ext in supported_extensions:
+            for file_path in data_room_path_obj.rglob(f"*{ext}"):
+                if file_path.is_file():
+                    try:
+                        rel_path = file_path.relative_to(data_room_path_obj)
+                        documents_dict[str(file_path)] = {
+                            'name': file_path.name,
+                            'path': str(rel_path),
+                            'content': f"[Indexed - {file_path.stat().st_size:,} bytes]",
+                            'metadata': {
+                                'source': str(file_path),
+                                'name': file_path.name,
+                                'path': str(rel_path)
+                            }
+                        }
+                    except ValueError:
+                        # Skip files outside data room path
+                        continue
+
+        if not documents_dict:
+            raise create_processing_error(
+                f"No supported documents found in {data_room_path}",
+                recovery_hint="Please ensure the data room contains PDF, DOCX, or text files"
+            )
+
+        return documents_dict
+
+    def _extract_chunks_from_faiss(self, document_processor) -> List[Dict]:
+        """Extract chunk information from loaded FAISS store"""
+        chunks = []
+
+        if not document_processor.vector_store:
+            logger.warning("No vector store available for chunk extraction")
+            return chunks
+
+        try:
+            # Access the docstore to get document metadata
+            docstore = document_processor.vector_store.docstore
+
+            for doc_id in docstore._dict.keys():
+                doc = docstore._dict[doc_id]
+                chunk_text = doc.page_content
+                if len(chunk_text) > 500:
+                    chunk_text = chunk_text[:500] + "..."
+
+                chunk_dict = {
+                    'text': chunk_text,
+                    'source': doc.metadata.get('name', ''),
+                    'path': doc.metadata.get('path', ''),
+                    'full_path': doc.metadata.get('source', ''),
+                    'metadata': doc.metadata
+                }
+                chunks.append(chunk_dict)
+
+        except (DocumentProcessingError, FileOperationError) as e:
+            ErrorHandler.handle_error(
+                e,
+                "Failed to extract chunks from FAISS store",
+                recovery_hint="The FAISS index may be corrupted"
+            )
+            # Fallback: create minimal chunks
+            chunks = [{
+                'text': '[Content available in search]',
+                'source': 'indexed_content',
+                'path': '',
+                'full_path': '',
+                'metadata': {}
+            }]
+
+        return chunks
+
+    def get_document_processor(self, store_name: str = None):
+        """
+        Get a configured document processor.
+
+        Args:
+            store_name: Optional store name for the processor
+
+        Returns:
+            Configured DocumentProcessor instance
+        """
+        from app.core.utils import create_document_processor
+        return create_document_processor(store_name=store_name)
+
+    def validate_data_room(self, data_room_path: str) -> bool:
+        """
+        Validate that a data room path exists and contains documents.
+
+        Args:
+            data_room_path: Path to validate
+
+        Returns:
+            True if valid, False otherwise
+        """
+        path_obj = Path(data_room_path)
+        if not path_obj.exists():
+            return False
+
+        return self._has_supported_files(path_obj)
+
+    def _has_supported_files(self, path_obj: Path) -> bool:
+        """
+        Check if path contains files with supported extensions.
+
+        Args:
+            path_obj: Path object to check
+
+        Returns:
+            True if supported files are found
+        """
+        from app.core import get_config
+        config = get_config()
+        supported_extensions = config.get_supported_extensions()
+
+        for ext in supported_extensions:
+            if list(path_obj.rglob(f"*{ext}")):
+                return True
+
+        return False

app/handlers/export_handler.py

@@ -0,0 +1,153 @@
+#!/usr/bin/env python3
+"""
+Export Handler
+
+Handles report export operations.
+"""
+
+from pathlib import Path
+
+from app.ui.session_manager import SessionManager
+from app.core.exceptions import ProcessingError
+from app.ui.error_handler import handle_ui_errors
+from app.core.exceptions import create_processing_error
+
+
+class ExportHandler:
+    """
+    Export handler that manages report export operations.
+    """
+
+    def __init__(self, session: SessionManager):
+        """Initialize handler with session manager"""
+        self.session = session
+
+    @handle_ui_errors("Export overview report", "Please ensure overview analysis is complete")
+    def export_overview_report(self) -> tuple[str, str]:
+        """
+        Export company overview report.
+
+        Returns:
+            Tuple of (file_name, content)
+        """
+        if not self.session.overview_summary:
+            raise create_processing_error(
+                "No overview analysis available for export",
+                recovery_hint="Please complete the overview analysis first"
+            )
+
+        company_name = self._get_company_name()
+        file_name = f"company_overview_{company_name}.md"
+        content = f"# Company Overview\n\n{self.session.overview_summary}"
+
+        return file_name, content
+
+    @handle_ui_errors("Export strategic report", "Please ensure strategic analysis is complete")
+    def export_strategic_report(self) -> tuple[str, str]:
+        """
+        Export strategic analysis report.
+
+        Returns:
+            Tuple of (file_name, content)
+        """
+        if not self.session.strategic_summary:
+            raise create_processing_error(
+                "No strategic analysis available for export",
+                recovery_hint="Please complete the strategic analysis first"
+            )
+
+        company_name = self._get_company_name()
+        file_name = f"dd_report_{company_name}.md"
+
+        content = "# Due Diligence Report\n\n"
+        if self.session.overview_summary:
+            content += f"## Company Overview\n\n{self.session.overview_summary}\n\n"
+        content += f"## Strategic Analysis\n\n{self.session.strategic_summary}"
+
+        return file_name, content
+
+    @handle_ui_errors("Export combined report", "Please ensure analysis is complete")
+    def export_combined_report(self) -> tuple[str, str]:
+        """
+        Export combined due diligence report.
+
+        Returns:
+            Tuple of (file_name, content)
+        """
+        if not (self.session.overview_summary or self.session.strategic_summary):
+            raise create_processing_error(
+                "No analysis data available for export",
+                recovery_hint="Please complete overview or strategic analysis first"
+            )
+
+        company_name = self._get_company_name()
+        file_name = f"complete_dd_report_{company_name}.md"
+
+        content = f"# Complete Due Diligence Report - {company_name.title()}\n\n"
+
+        if self.session.overview_summary:
+            content += f"## Company Overview\n\n{self.session.overview_summary}\n\n"
+
+        if self.session.strategic_summary:
+            content += f"## Strategic Analysis\n\n{self.session.strategic_summary}\n\n"
+
+        # Add checklist results if available
+        if self.session.checklist_results:
+            content += "## Checklist Analysis\n\n"
+            for category, items in self.session.checklist_results.items():
+                content += f"### {category}\n\n"
+                if isinstance(items, list):
+                    for item in items:
+                        if isinstance(item, dict):
+                            content += f"- {item.get('text', str(item))}\n"
+                        else:
+                            content += f"- {str(item)}\n"
+                content += "\n"
+
+        # Add question answers if available
+        if self.session.question_answers:
+            content += "## Due Diligence Questions\n\n"
+            for question, answer in self.session.question_answers.items():
+                if isinstance(answer, dict) and answer.get('has_answer'):
+                    content += f"### {question}\n\n{answer.get('answer', '')}\n\n"
+
+        return file_name, content
+
+    @handle_ui_errors("Export checklist report", "Please ensure checklist analysis is complete")
+    def export_checklist_report(self) -> tuple[str, str]:
+        """
+        Export checklist analysis report.
+
+        Returns:
+            Tuple of (file_name, content)
+        """
+        if not self.session.checklist_results:
+            raise create_processing_error(
+                "No checklist results available for export",
+                recovery_hint="Please complete the checklist analysis first"
+            )
+
+        company_name = self._get_company_name()
+        file_name = f"checklist_analysis_{company_name}.md"
+
+        content = f"# Checklist Analysis Report - {company_name.title()}\n\n"
+
+        for category, items in self.session.checklist_results.items():
+            content += f"## {category}\n\n"
+            if isinstance(items, list):
+                for item in items:
+                    if isinstance(item, dict):
+                        content += f"- {item.get('text', str(item))}\n"
+                    else:
+                        content += f"- {str(item)}\n"
+            content += "\n"
+
+        return file_name, content
+
+    def _get_company_name(self) -> str:
+        """Get company name from current documents"""
+        documents = self.session.documents
+        if documents:
+            company_name = Path(list(documents.keys())[0]).parent.name
+            return company_name
+        return 'export'

app/main.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env python3
+"""
+Main Application Entry Point
+"""
+
+# Standard library imports
+import os
+import warnings
+
+# Third-party imports
+import streamlit as st
+
+# Local imports
+from app.core.config import init_app_config
+from app.core.logging import configure_langchain_logging
+from app.handlers.ai_handler import AIHandler
+from app.handlers.document_handler import DocumentHandler
+from app.handlers.export_handler import ExportHandler
+from app.ui.session_manager import SessionManager
+from app.ui.sidebar import Sidebar
+from app.ui.tabs.checklist_tab import ChecklistTab
+from app.ui.tabs.graph_tab import GraphTab
+from app.ui.tabs.overview_tab import OverviewTab
+from app.ui.tabs.qa_tab import QATab
+from app.ui.tabs.questions_tab import QuestionsTab
+from app.ui.tabs.strategic_tab import StrategicTab
+
+# Enable tokenizers parallelism for better performance
+os.environ.setdefault("TOKENIZERS_PARALLELISM", "true")
+
+# Initialize for Streamlit Cloud deployment (must be done before other imports)
+try:
+    from scripts.streamlit_cloud_config import initialize_for_streamlit_cloud
+    initialize_for_streamlit_cloud()
+except ImportError:
+    # Local development - skip cloud initialization
+    pass
+
+# Only suppress specific known non-critical warnings
+warnings.filterwarnings("ignore", message=".*Relevance scores must be between.*")
+warnings.filterwarnings("ignore", message=".*No relevant docs were retrieved.*")
+
+
+class App:
+    """Main application class that orchestrates all components."""
+
+    def __init__(self):
+        """Initialize the application"""
+        # Initialize configuration
+        self.config = init_app_config()
+
+        # Initialize session manager
+        self.session = SessionManager()
+
+        # Initialize handlers
+        self.document_handler = DocumentHandler(self.session)
+        self.ai_handler = AIHandler(self.session)
+        self.export_handler = ExportHandler(self.session)
+
+        # Initialize UI components
+        self.sidebar = Sidebar(self.session, self.config)
+        self.tabs = {
+            'overview': OverviewTab(self.session, self.config, self.ai_handler, self.export_handler),
+            'strategic': StrategicTab(self.session, self.config, self.ai_handler, self.export_handler),
+            'checklist': ChecklistTab(self.session, self.config, self.ai_handler),
+            'questions': QuestionsTab(self.session, self.config, self.ai_handler),
+            'qa': QATab(self.session, self.config, self.ai_handler),
+            'graph': GraphTab(self.session, self.config, self.ai_handler, self.export_handler)
+        }
+
+        # Configure Streamlit page
+        st.set_page_config(
+            page_title=self.config.ui['page_title'],
+            page_icon=self.config.ui['page_icon'],
+            layout=self.config.ui['layout']
+        )
+
+    def run(self):
+        """Run the main application"""
+        # Render header
+        st.title("🤖 AI Due Diligence")
+        st.markdown("**Intelligent M&A Analysis:** Strategic assessment, automated document review, and AI-powered insights")
+
+        # Render sidebar and get selections
+        data_room_path, process_button = self.sidebar.render()
+
+        # Store the selected data room path
+        if data_room_path:
+            self.session.data_room_path = data_room_path
+
+        # Main tabs
+        tab_names = [
+            "🏢 Company Overview",
+            "🎯 Strategic Analysis",
+            "📊 Checklist Matching",
+            "❓ Due Diligence Questions",
+            "💬 Q&A with Citations",
+            "🧠 Knowledge Graph"
+        ]
+
+        tabs = st.tabs(tab_names)
+
+        with tabs[0]:
+            self.tabs['overview'].render()
+
+        with tabs[1]:
+            self.tabs['strategic'].render()
+
+        with tabs[2]:
+            self.tabs['checklist'].render()
+
+        with tabs[3]:
+            self.tabs['questions'].render()
+
+        with tabs[4]:
+            self.tabs['qa'].render()
+
+        with tabs[5]:
+            self.tabs['graph'].render()
+
+        # Processing trigger
+        if process_button and data_room_path:
+            with st.spinner("🚀 Processing data room..."):
+                self.sidebar.process_data_room(data_room_path)
+
+
+def main():
+    """Main application entry point"""
+    # Configure LangChain logging
+    configure_langchain_logging(log_level="WARNING")
+
+    try:
+        app = App()
+        app.run()
+    except Exception as e:
+        from app.ui.error_handler import ErrorHandler
+        ErrorHandler.handle_error(
+            e,
+            "Application startup failed",
+            recovery_hint="Please refresh the page and try again"
+        )
+        st.stop()
+
+
+if __name__ == "__main__":
+    main()

app/services/ai_client.py

@@ -0,0 +1,301 @@
+#!/usr/bin/env python3
+"""
+AI Client
+
+Handles Anthropic API client and LLM interaction logic.
+Provides clean interface for LLM operations and connection management.
+"""
+
+from typing import Optional, Any, List
+
+from app.core.exceptions import AIError
+from app.services.ai_config import AIConfig
+from app.core.exceptions import LLMConnectionError, LLMAuthenticationError, LLMTimeoutError, LLMQuotaExceededError, LLMInvalidResponseError
+
+# Import specific exception types for robust error handling
+try:
+    from anthropic import (
+        APIConnectionError, APIError, APITimeoutError, AuthenticationError,
+        BadRequestError, ConflictError, InternalServerError, NotFoundError,
+        PermissionDeniedError, RateLimitError, UnprocessableEntityError,
+        ServiceUnavailableError
+    )
+except ImportError:
+    # Fallback if anthropic package is not directly available
+    APIConnectionError = APIError = APITimeoutError = AuthenticationError = None
+    BadRequestError = ConflictError = InternalServerError = NotFoundError = None
+    PermissionDeniedError = RateLimitError = UnprocessableEntityError = None
+    ServiceUnavailableError = None
+
+
+class AIClient:
+    """
+    Anthropic API client for LLM interactions.
+
+    This class manages the connection to Anthropic's Claude models,
+    handles initialization, and provides methods for LLM operations.
+    """
+
+    def __init__(self, config: AIConfig) -> None:
+        """
+        Initialize AI client with configuration.
+
+        Args:
+            config: AIConfig object containing service configuration
+
+        Raises:
+            AIError: If initialization fails
+        """
+        self.config: AIConfig = config
+        self._llm: Optional[Any] = None
+        self._initialized: bool = False
+
+    def _ensure_initialized(self) -> None:
+        """
+        Ensure the AI client is properly initialized and ready for use.
+
+        This method handles lazy initialization of the AI client, creating
+        the underlying LLM connection and testing it with a simple query.
+
+        Raises:
+            AIError: If initialization fails due to configuration or connection issues
+        """
+        if self._initialized:
+            return
+
+        try:
+            from langchain_anthropic import ChatAnthropic
+
+            self._llm = ChatAnthropic(
+                api_key=self.config.api_key,
+                model=self.config.model,
+                temperature=self.config.temperature,
+                max_tokens=self.config.max_tokens
+            )
+
+            # Test the connection with a simple query that validates AI functionality
+            from langchain_core.messages import HumanMessage
+            test_response = self._llm.invoke([
+                HumanMessage(content="Please respond with 'AI connection successful' if you can read this message.")
+            ])
+            if not test_response or not hasattr(test_response, 'content') or not test_response.content.strip():
+                raise AIError("AI service test failed - no valid response received")
+
+            # Verify the response contains expected content
+            response_text = test_response.content.strip().lower()
+            if "successful" not in response_text and "ai" not in response_text:
+                raise AIError("AI service test failed - unexpected response format")
+
+            self._initialized = True
+
+        except ImportError as e:
+            raise AIError(
+                f"Missing required AI library: {str(e)}",
+                user_message="AI libraries not installed",
+                recovery_hint="Please install required dependencies"
+            )
+        except Exception as e:
+            self._handle_llm_error(e)
+
+    def _handle_llm_error(self, error: Exception, include_invalid_response: bool = False) -> None:
+        """
+        Handle LLM-related errors with robust error type detection.
+
+        This method uses exception type checking as the primary classification method,
+        with string-based fallbacks for compatibility with different library versions.
+
+        Args:
+            error: The exception that occurred
+            include_invalid_response: Whether to include invalid response error handling
+
+        Raises:
+            Specific LLM error types based on exception type and content
+        """
+        # Primary: Check exception types for robust classification
+        if self._is_authentication_error(error):
+            raise LLMAuthenticationError(
+                f"AI authentication failed: {str(error)}",
+                user_message="AI authentication failed",
+                recovery_hint="Please check your API key"
+            )
+        elif self._is_timeout_error(error):
+            raise LLMTimeoutError(
+                f"AI service timeout: {str(error)}",
+                user_message="AI service timed out",
+                recovery_hint="Please try again later"
+            )
+        elif self._is_quota_error(error):
+            raise LLMQuotaExceededError(
+                f"AI quota exceeded: {str(error)}",
+                user_message="AI quota exceeded",
+                recovery_hint="Please check your API usage limits"
+            )
+        elif self._is_connection_error(error):
+            raise LLMConnectionError(
+                f"AI connection failed: {str(error)}",
+                user_message="AI connection failed",
+                recovery_hint="Please check your network connection"
+            )
+        elif include_invalid_response and self._is_invalid_response_error(error):
+            raise LLMInvalidResponseError(
+                f"AI returned invalid response: {str(error)}",
+                user_message="AI returned invalid response",
+                recovery_hint="Please try again"
+            )
+
+        # Default error messages based on context
+        if include_invalid_response:
+            raise AIError(
+                f"Response generation failed: {str(error)}",
+                user_message="Failed to generate AI response",
+                recovery_hint="Please try again or check your API key"
+            )
+        else:
+            raise AIError(
+                f"Failed to initialize AI client: {str(error)}",
+                user_message="AI client initialization failed",
+                recovery_hint="Please check your API key and network connection"
+            )
+
+    def _is_authentication_error(self, error: Exception) -> bool:
+        """Check if error is an authentication-related error."""
+        # Primary: Check exception types
+        if AuthenticationError and isinstance(error, AuthenticationError):
+            return True
+        if PermissionDeniedError and isinstance(error, PermissionDeniedError):
+            return True
+
+        # Fallback: String-based detection for compatibility
+        error_msg = str(error).lower()
+        return "authentication" in error_msg or "api key" in error_msg or "unauthorized" in error_msg
+
+    def _is_timeout_error(self, error: Exception) -> bool:
+        """Check if error is a timeout-related error."""
+        # Primary: Check exception types
+        if APITimeoutError and isinstance(error, APITimeoutError):
+            return True
+
+        # Fallback: String-based detection
+        error_msg = str(error).lower()
+        return "timeout" in error_msg or "timed out" in error_msg
+
+    def _is_quota_error(self, error: Exception) -> bool:
+        """Check if error is a quota/rate limit related error."""
+        # Primary: Check exception types
+        if RateLimitError and isinstance(error, RateLimitError):
+            return True
+
+        # Fallback: String-based detection
+        error_msg = str(error).lower()
+        return "quota" in error_msg or "rate limit" in error_msg or "limit exceeded" in error_msg
+
+    def _is_connection_error(self, error: Exception) -> bool:
+        """Check if error is a connection/network related error."""
+        # Primary: Check exception types
+        if APIConnectionError and isinstance(error, APIConnectionError):
+            return True
+        if InternalServerError and isinstance(error, InternalServerError):
+            return True
+        if ServiceUnavailableError and isinstance(error, ServiceUnavailableError):
+            return True
+
+        # Fallback: String-based detection
+        error_msg = str(error).lower()
+        return ("connection" in error_msg or "network" in error_msg or
+                "connection reset" in error_msg or "connection refused" in error_msg)
+
+    def _is_invalid_response_error(self, error: Exception) -> bool:
+        """Check if error is related to invalid/malformed responses."""
+        # Primary: Check exception types
+        if BadRequestError and isinstance(error, BadRequestError):
+            return True
+        if UnprocessableEntityError and isinstance(error, UnprocessableEntityError):
+            return True
+
+        # Fallback: String-based detection
+        error_msg = str(error).lower()
+        return ("invalid" in error_msg or "malformed" in error_msg or
+                "bad request" in error_msg or "unprocessable" in error_msg)
+
+    @property
+    def is_available(self) -> bool:
+        """
+        Check if AI client is available and ready for operations.
+
+        This property performs lazy initialization if needed and returns
+        the availability status of the AI client.
+
+        Returns:
+            True if AI client is initialized and ready, False otherwise
+        """
+        try:
+            self._ensure_initialized()
+            return True
+        except (AIError):
+            return False
+
+    @property
+    def llm(self) -> Any:
+        """
+        Get the underlying LLM instance for direct access.
+
+        This property provides access to the raw LangChain LLM object
+        for advanced use cases that require direct interaction.
+
+        Returns:
+            LangChain LLM instance (ChatAnthropic)
+
+        Raises:
+            AIError: If LLM is not initialized
+        """
+        self._ensure_initialized()
+        return self._llm
+
+    def generate_response(self, messages: List[dict]) -> str:
+        """
+        Generate a response using the LLM.
+
+        Args:
+            messages: List of message dictionaries for the LLM
+
+        Returns:
+            Generated response content
+
+        Raises:
+            AIError: If response generation fails
+        """
+        self._ensure_initialized()
+
+        try:
+            response = self._llm.invoke(messages)
+            return response.content.strip()
+        except Exception as e:
+            self._handle_llm_error(e, include_invalid_response=True)
+
+    def generate_text(self, prompt: str, context: Optional[List[str]] = None) -> str:
+        """
+        Generate text using the AI client.
+
+        Args:
+            prompt: The main prompt for text generation
+            context: Optional context documents
+
+        Returns:
+            Generated text response
+        """
+        self._ensure_initialized()
+
+        # Prepare the full prompt
+        full_prompt = prompt
+        if context:
+            context_str = "\n\n".join(context[:3])  # Limit context to prevent token overflow
+            full_prompt = f"Context:\n{context_str}\n\n{prompt}"
+
+        try:
+            from langchain_core.messages import HumanMessage
+
+            response = self._llm.invoke([HumanMessage(content=full_prompt)])
+            return response.content.strip()
+
+        except Exception as e:
+            self._handle_llm_error(e, include_invalid_response=True)

app/services/ai_config.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+"""
+AI Configuration
+
+Configuration settings for AI service operations.
+Provides type safety and validation for AI service parameters.
+"""
+
+from dataclasses import dataclass
+
+from app.core.exceptions import ConfigError
+from app.core.constants import TEMPERATURE
+
+
+@dataclass
+class AIConfig:
+    """
+    Configuration settings for AI service operations.
+
+    This dataclass encapsulates all configuration parameters needed
+    for AI service initialization and operation, providing type safety
+    and validation.
+
+    Attributes:
+        api_key: Anthropic API key for authentication
+        model: Claude model name to use for operations
+        temperature: Sampling temperature (0.0 = deterministic, higher = more creative)
+        max_tokens: Maximum tokens to generate in responses
+
+    Example:
+        config = AIConfig(
+            api_key="sk-ant-...",
+            model="claude-3-5-sonnet",
+            temperature=TEMPERATURE,
+            max_tokens=4000
+        )
+    """
+    api_key: str
+    model: str
+    temperature: float = TEMPERATURE
+    max_tokens: int = 4000
+
+    def validate(self) -> None:
+        """
+        Validate the AI configuration for required values and consistency.
+
+        Performs comprehensive validation of all configuration parameters
+        to ensure they are valid for AI service operations.
+
+        Raises:
+            ConfigError: If any configuration values are invalid
+        """
+        if not self.api_key or not self.api_key.strip():
+            raise ConfigError(
+                "AI API key is missing",
+                user_message="API key is required for AI features",
+                recovery_hint="Please configure your Anthropic API key in the sidebar"
+            )
+
+        if not self.model or not self.model.strip():
+            raise ConfigError(
+                "AI model is not specified",
+                user_message="AI model selection is required",
+                recovery_hint="Please select a Claude model"
+            )

app/services/ai_service.py

@@ -0,0 +1,438 @@
+#!/usr/bin/env python3
+"""
+AI Service
+
+Provides a clean interface for AI operations.
+Reduces coupling between AI components and the rest of the system.
+"""
+
+from typing import Optional, Dict, List, Any
+
+from app.core.exceptions import AIError, ConfigError
+# Removed circular import: from app.ui.error_handler import handle_processing_errors
+from app.core.exceptions import create_config_error
+from app.core.constants import QA_MAX_TOKENS, SUPPORTED_ANALYSIS_TYPES
+from app.services.ai_config import AIConfig
+from app.services.ai_client import AIClient
+from app.services.response_parser import ResponseParser
+
+
+class AIService:
+    """
+    Simplified AI service providing clean, type-safe interface for AI operations.
+
+    This service replaces the complex DDChecklistAgent with a focused, simple interface
+    that handles the core AI operations needed by the application. It provides:
+
+    Features:
+    - Type-safe AI operations with comprehensive error handling
+    - Multiple analysis types (overview, strategic, checklist, questions)
+    - Token usage estimation and limits
+    - Configurable AI models and parameters
+    - Clean separation of concerns
+
+    Attributes:
+        config: AIConfig object containing service configuration
+        is_available: Property indicating if service is ready for use
+
+    Example:
+        config = AIConfig(api_key="sk-ant-...", model="claude-3-sonnet-20240229")
+        ai_service = AIService(config)
+
+        if ai_service.is_available:
+            result = ai_service.analyze_documents(docs, "overview")
+            answer = ai_service.answer_question("What is the revenue?", context)
+    """
+
+    def __init__(self, config: AIConfig) -> None:
+        """
+        Initialize AI service with configuration and validate setup.
+
+        Args:
+            config: AIConfig object containing service configuration
+
+        Raises:
+            ConfigError: If configuration validation fails
+        """
+        self.config: AIConfig = config
+        self.config.validate()
+        self._client: Optional[AIClient] = None
+
+    @property
+    def _ensure_client(self) -> AIClient:
+        """
+        Ensure the AI client is properly initialized.
+
+        Returns:
+            Initialized AIClient instance
+
+        Raises:
+            AIError: If client initialization fails
+        """
+        if self._client is None:
+            self._client = AIClient(self.config)
+        return self._client
+
+    @property
+    def is_available(self) -> bool:
+        """
+        Check if AI service is available and ready for operations.
+
+        This property performs lazy initialization if needed and returns
+        the availability status of the AI service.
+
+        Returns:
+            True if AI service is initialized and ready, False otherwise
+        """
+        try:
+            return self._ensure_client.is_available
+        except (AIError, ConfigError):
+            return False
+
+    @property
+    def llm(self) -> Any:
+        """
+        Get the underlying LLM instance for direct access.
+
+        This property provides access to the raw LangChain LLM object
+        for advanced use cases that require direct interaction.
+
+        Returns:
+            LangChain LLM instance (ChatAnthropic)
+
+        Raises:
+            AIError: If LLM is not initialized
+        """
+        return self._ensure_client.llm
+
+    # Removed decorator to avoid circular imports
+    def generate_text(
+        self,
+        prompt: str,
+        context: Optional[List[str]] = None,
+        max_length: Optional[int] = None
+    ) -> str:
+        """
+        Generate text using the AI service.
+
+        Args:
+            prompt: The main prompt for text generation
+            context: Optional context documents
+            max_length: Maximum response length
+
+        Returns:
+            Generated text response
+        """
+        client = self._ensure_client
+        response = client.generate_text(prompt, context)
+        return ResponseParser.format_response(response, max_length)
+
+    # Removed decorator to avoid circular imports
+    def analyze_documents(
+        self,
+        documents: Dict[str, Dict[str, Any]],
+        analysis_type: str,
+        strategy_text: Optional[str] = None,
+        checklist_results: Optional[Dict[str, Any]] = None
+    ) -> str:
+        """
+        Analyze documents using AI with different analysis types.
+
+        This method performs comprehensive document analysis using AI, supporting
+        multiple analysis types for different business use cases.
+
+        Args:
+            documents: Dictionary mapping document names to document data.
+                      Each document dict should contain 'content' and other metadata.
+            analysis_type: Type of analysis to perform. Supported types:
+                         - "overview": Company overview and business analysis
+                         - "strategic": Strategic positioning and recommendations
+                         - "checklist": Due diligence checklist analysis
+                         - "questions": Answer due diligence questions
+            strategy_text: Optional strategy document content for context
+            checklist_results: Optional existing checklist results for strategic analysis
+
+        Returns:
+            AI-generated analysis text with comprehensive insights
+
+        Raises:
+            AIError: If analysis fails or service is unavailable
+            ValueError: If analysis_type is not supported
+
+        Example:
+            docs = {
+                "annual_report.pdf": {"content": "Company financials...", "name": "Annual Report"},
+                "strategy.docx": {"content": "Strategic plan...", "name": "Strategy"}
+            }
+            analysis = ai_service.analyze_documents(docs, "overview")
+        """
+        # Input validation
+        if not documents:
+            raise ValueError("Documents dictionary cannot be None or empty")
+
+        if not isinstance(documents, dict):
+            raise ValueError("Documents must be a dictionary")
+
+        if analysis_type not in SUPPORTED_ANALYSIS_TYPES:
+            raise ValueError(f"Invalid analysis type: {analysis_type}. Supported types: {SUPPORTED_ANALYSIS_TYPES}")
+
+        # Validate each document has content
+        for doc_name, doc_data in documents.items():
+            if not isinstance(doc_data, dict):
+                raise ValueError(f"Document '{doc_name}' must be a dictionary")
+            if 'content' not in doc_data:
+                raise ValueError(f"Document '{doc_name}' must contain a 'content' key")
+            if not doc_data['content']:
+                raise ValueError(f"Document '{doc_name}' content cannot be empty")
+
+        # Prepare context from documents
+        context_docs = ResponseParser.prepare_context_documents(documents)
+
+        # Create analysis prompt based on type
+        prompt = self._get_analysis_prompt(analysis_type, context_docs, strategy_text, checklist_results)
+
+        return self.generate_text(prompt, max_length=3000)
+
+    def _get_analysis_prompt(self, analysis_type: str, context_docs: List[str],
+                           strategy_text: Optional[str] = None,
+                           checklist_results: Optional[Dict[str, Any]] = None) -> str:
+        """
+        Get the appropriate analysis prompt based on analysis type.
+
+        Args:
+            analysis_type: Type of analysis to perform
+            context_docs: Prepared context documents
+            strategy_text: Optional strategy document content
+            checklist_results: Optional existing checklist results
+
+        Returns:
+            Generated prompt for the specified analysis type
+
+        Raises:
+            ValueError: If analysis_type is not supported
+        """
+        if analysis_type == "overview":
+            return ResponseParser.create_overview_prompt(context_docs, strategy_text, checklist_results)
+        if analysis_type == "strategic":
+            return ResponseParser.create_strategic_prompt(context_docs, strategy_text, checklist_results)
+        if analysis_type == "checklist":
+            return ResponseParser.create_checklist_prompt(context_docs)
+        if analysis_type == "questions":
+            return ResponseParser.create_questions_prompt(context_docs)
+
+        raise ValueError(f"Unknown analysis type: {analysis_type}")
+
+
+    # Removed decorator to avoid circular imports
+    def answer_question(
+        self,
+        question: str,
+        context_docs: List[str],
+        max_length: Optional[int] = None
+    ) -> str:
+        """
+        Answer a specific question using AI with document context.
+
+        This method performs question answering by analyzing the provided
+        question against relevant document excerpts to provide accurate,
+        context-aware answers.
+
+        Args:
+            question: The question to answer. Should be clear and specific
+                     for best results (e.g., "What is the company's revenue?"
+                     rather than "Tell me about revenue").
+            context_docs: List of relevant document excerpts that may contain
+                         information to answer the question. Should be
+                         pre-filtered to most relevant content.
+            max_length: Optional maximum length for the answer in characters.
+                      If None, uses service default (typically 2000 chars).
+
+        Returns:
+            AI-generated answer with citations and context where applicable
+
+        Raises:
+            AIError: If question answering fails or service is unavailable
+
+        Example:
+            context = [
+                "The company reported $50M in revenue for Q4 2023...",
+                "Revenue growth was 15% compared to previous year..."
+            ]
+            answer = ai_service.answer_question(
+                "What was the company's revenue for Q4 2023?",
+                context
+            )
+        """
+        # Input validation
+        if not question or not isinstance(question, str):
+            raise ValueError("Question must be a non-empty string")
+
+        if not context_docs:
+            raise ValueError("Context documents list cannot be None or empty")
+
+        if not isinstance(context_docs, list):
+            raise ValueError("Context documents must be a list")
+
+        # Validate each context document
+        for i, doc in enumerate(context_docs):
+            if not isinstance(doc, str):
+                raise ValueError(f"Context document at index {i} must be a string")
+            if not doc.strip():
+                raise ValueError(f"Context document at index {i} cannot be empty or whitespace only")
+
+        prompt = ResponseParser.create_question_answer_prompt(question, context_docs)
+        return self.generate_text(prompt, max_length=max_length or QA_MAX_TOKENS)
+
+    def get_token_usage_estimate(self, text: str) -> int:
+        """
+        Estimate token usage for a given text using character-based approximation.
+
+        This method provides a rough estimate of token count based on character
+        length. Actual token counts may vary depending on the specific tokenizer
+        used by the AI model.
+
+        Args:
+            text: Text to estimate token count for. Can be any string content
+                 including prompts, documents, or responses.
+
+        Returns:
+            Estimated token count (integer). Uses approximation of ~4 characters
+            per token, which is typical for English text with Claude models.
+
+        Note:
+            This is an approximation. For precise token counting, use the
+            actual tokenizer for the specific AI model being used.
+
+        Example:
+            estimate = ai_service.get_token_usage_estimate("Hello, how are you?")
+            # Returns approximately 5-6 tokens
+        """
+        if not text:
+            return 0
+
+        # Rough estimation: ~4 characters per token for English text
+        # This is a conservative estimate that works well for Claude models
+        return len(text) // 4
+
+    def is_within_token_limit(self, text: str, max_tokens: int = 100000) -> bool:
+        """
+        Check if text is within specified token limits.
+
+        This method helps prevent token overflow by checking if the estimated
+        token count for a given text is within acceptable limits.
+
+        Args:
+            text: Text to check for token limit compliance
+            max_tokens: Maximum allowed tokens. Default is 100,000 which is
+                       a conservative limit for most AI models.
+
+        Returns:
+            True if estimated token count is within the specified limit,
+            False if it exceeds the limit.
+
+        Note:
+            Uses character-based estimation which may not be perfectly accurate.
+            For critical token limit checking, consider using the actual tokenizer.
+
+        Example:
+            if ai_service.is_within_token_limit(document_content, 8000):
+                # Safe to process
+                analysis = ai_service.analyze_documents(docs, "overview")
+            else:
+                # Need to truncate or split content
+                print("Content too long for processing")
+        """
+        if not text:
+            return True
+
+        estimated_tokens = self.get_token_usage_estimate(text)
+        return estimated_tokens <= max_tokens
+
+
+# Factory function for easy service creation
+def create_ai_service(
+    api_key: str,
+    model: str,
+    temperature: float = 0.1,
+    max_tokens: int = 4000
+) -> AIService:
+    """
+    Create and configure an AI service instance with the given parameters.
+
+    This factory function provides a convenient way to create AI service instances
+    with proper configuration and validation. It handles all the setup steps
+    including configuration validation and service initialization.
+
+    Args:
+        api_key: Anthropic API key for authentication. Must be a valid
+                Anthropic API key with sufficient permissions.
+        model: Claude model to use for AI operations. Examples:
+              - "claude-3-5-sonnet" (recommended for most use cases)
+              - "claude-3-5-haiku-20241022" (faster, less expensive)
+              - "claude-3-opus-20240229" (most capable, more expensive)
+        temperature: Sampling temperature for response generation (0.0 to 1.0).
+                   Lower values (0.1) produce more deterministic responses.
+                   Higher values (0.7+) produce more creative responses.
+        max_tokens: Maximum tokens to generate in AI responses.
+                   Default 4000 tokens provides good balance of length and cost.
+
+    Returns:
+        Fully configured and validated AIService instance ready for use
+
+    Raises:
+        ConfigError: If configuration parameters are invalid
+        AIError: If AI service initialization fails
+
+    Example:
+        # Basic usage
+        ai_service = create_ai_service("sk-ant-...", "claude-3-5-sonnet")
+
+        # Advanced configuration
+        ai_service = create_ai_service(
+            api_key="sk-ant-...",
+            model="claude-3-5-haiku-20241022",
+            temperature=0.2,
+            max_tokens=QA_MAX_TOKENS
+        )
+
+        # Use the service
+        if ai_service.is_available:
+            answer = ai_service.answer_question("What is AI?", ["AI is artificial intelligence..."])
+    """
+    # Validate and resolve API key
+    api_key = _resolve_api_key(api_key)
+
+    config = AIConfig(
+        api_key=api_key, 
+        model=model, 
+        temperature=temperature, 
+        max_tokens=max_tokens
+    )
+    return AIService(config)
+
+
+def _resolve_api_key(api_key: Optional[str]) -> str:
+    """
+    Resolve API key from parameter or environment variable.
+
+    Args:
+        api_key: API key provided by user, or None
+
+    Returns:
+        Resolved API key string
+
+    Raises:
+        ConfigError: If no API key is available
+    """
+    if api_key is not None:
+        return api_key
+
+    import os
+    env_key = os.getenv('ANTHROPIC_API_KEY')
+    if env_key is not None:
+        return env_key
+
+    raise create_config_error(
+        "AI API key is missing",
+        recovery_hint="Please set ANTHROPIC_API_KEY environment variable or pass api_key parameter"
+    )

app/services/response_parser.py

@@ -0,0 +1,185 @@
+#!/usr/bin/env python3
+"""
+Response Parser
+
+Handles response parsing and formatting functions for AI operations.
+Provides methods for creating prompts and processing AI responses.
+"""
+
+from typing import List, Dict, Any, Optional
+
+from app.core.exceptions import ProcessingError
+
+
+class ResponseParser:
+    """
+    Parser for AI responses and prompt generation.
+
+    This class provides methods for creating structured prompts
+    and processing AI responses for different analysis types.
+    """
+
+    @staticmethod
+    def create_overview_prompt(
+        context_docs: List[str],
+        strategy_text: Optional[str],
+        checklist_results: Optional[Dict]
+    ) -> str:
+        """Create overview analysis prompt"""
+        prompt = "Based on the following company documents, provide a comprehensive overview analysis:\n\n"
+
+        if context_docs:
+            prompt += "Company Documents:\n" + "\n\n".join(context_docs) + "\n\n"
+
+        if strategy_text:
+            prompt += f"Strategic Context:\n{strategy_text[:1000]}\n\n"
+
+        if checklist_results:
+            prompt += f"Checklist Findings:\n{str(checklist_results)[:1000]}\n\n"
+
+        prompt += """Please provide:
+1. Company overview and business model
+2. Key strengths and competitive advantages
+3. Main risks and challenges
+4. Financial health indicators
+5. Strategic recommendations
+
+Be specific, factual, and focus on the most important insights."""
+
+        return prompt
+
+    @staticmethod
+    def create_strategic_prompt(
+        context_docs: List[str],
+        strategy_text: Optional[str],
+        checklist_results: Optional[Dict]
+    ) -> str:
+        """Create strategic analysis prompt"""
+        prompt = "Provide a strategic analysis based on the following company information:\n\n"
+
+        if strategy_text:
+            prompt += f"Strategic Framework:\n{strategy_text[:1000]}\n\n"
+
+        if context_docs:
+            prompt += "Company Documents:\n" + "\n\n".join(context_docs) + "\n\n"
+
+        if checklist_results:
+            prompt += f"Operational Findings:\n{str(checklist_results)[:1000]}\n\n"
+
+        prompt += """Please analyze:
+1. Strategic positioning and market opportunities
+2. Operational strengths and weaknesses
+3. Risk mitigation strategies
+4. Growth potential and recommendations
+5. Investment considerations
+
+Focus on strategic implications and actionable insights."""
+
+        return prompt
+
+    @staticmethod
+    def create_checklist_prompt(context_docs: List[str]) -> str:
+        """Create checklist analysis prompt"""
+        prompt = "Analyze the following documents against standard due diligence checklist items:\n\n"
+
+        if context_docs:
+            prompt += "Documents to Analyze:\n" + "\n\n".join(context_docs) + "\n\n"
+
+        prompt += """For each major due diligence category, identify:
+1. What information is available in the documents
+2. What information appears to be missing
+3. Any red flags or concerns identified
+4. Recommendations for further investigation
+
+Be thorough and specific in your analysis."""
+
+        return prompt
+
+    @staticmethod
+    def create_questions_prompt(context_docs: List[str]) -> str:
+        """Create questions analysis prompt"""
+        prompt = "Answer due diligence questions based on the following documents:\n\n"
+
+        if context_docs:
+            prompt += "Reference Documents:\n" + "\n\n".join(context_docs) + "\n\n"
+
+        prompt += """For each question, provide:
+1. Direct answer based on available information
+2. Supporting evidence from the documents
+3. Confidence level in the answer
+4. Any additional context or caveats
+
+If information is not available, clearly state this and suggest what additional information would be needed."""
+
+        return prompt
+
+    @staticmethod
+    def create_question_answer_prompt(question: str, context_docs: List[str]) -> str:
+        """Create prompt for answering a specific question"""
+        return f"""Based on the following document excerpts, please answer this question:
+
+Question: {question}
+
+Relevant Document Excerpts:
+{"\n\n".join(context_docs[:5])}
+
+Please provide a comprehensive, factual answer with specific references to the source documents.
+If the information is not available in the provided context, clearly state this."""
+
+    @staticmethod
+    def format_response(response: str, max_length: Optional[int] = None) -> str:
+        """
+        Format and clean AI response.
+
+        Args:
+            response: Raw AI response
+            max_length: Optional maximum length for the response
+
+        Returns:
+            Formatted response
+        
+        Raises:
+            ProcessingError: If response formatting fails
+        """
+        try:
+            if not response:
+                raise ValueError("Response cannot be empty")
+
+            result = response.strip()
+            if max_length and len(result) > max_length:
+                result = result[:max_length] + "..."
+            return result
+        except Exception as e:
+            raise ProcessingError(f"Failed to format AI response: {e}")
+
+    @staticmethod
+    def prepare_context_documents(documents: Dict[str, Dict[str, Any]], max_docs: int = 5) -> List[str]:
+        """
+        Prepare context documents for AI processing.
+
+        Args:
+            documents: Dictionary mapping document names to document data
+            max_docs: Maximum number of documents to process
+
+        Returns:
+            List of formatted document contexts
+            
+        Raises:
+            ProcessingError: If document preparation fails
+        """
+        try:
+            if not documents:
+                raise ValueError("No documents provided for context preparation")
+
+            context_docs = []
+            for doc_key, doc_data in list(documents.items())[:max_docs]:
+                if isinstance(doc_data, dict) and 'content' in doc_data:
+                    content = doc_data['content'][:1000]  # Truncate long content
+                    context_docs.append(f"Document: {doc_data.get('name', doc_key)}\n{content}")
+
+            if not context_docs:
+                raise ValueError("No valid documents found with content")
+
+            return context_docs
+        except Exception as e:
+            raise ProcessingError(f"Failed to prepare context documents: {e}")

app/ui/__init__.py

@@ -0,0 +1,9 @@
+"""
+UI Components Package
+
+Contains all user interface components and layout functions.
+"""
+
+from .sidebar import Sidebar
+
+__all__ = ['Sidebar']

app/ui/error_handler.py

@@ -0,0 +1,284 @@
+#!/usr/bin/env python3
+"""
+Standardized Error Handling System
+
+Provides consistent error handling patterns across all modules.
+Centralizes error logging, user messaging, and recovery mechanisms.
+"""
+
+import logging
+import streamlit as st
+from typing import Any, Optional, Callable, TypeVar
+from functools import wraps
+
+from app.core.exceptions import (
+    AppException, ValidationError, ProcessingError,
+    AIError, ConfigError
+)
+
+logger = logging.getLogger(__name__)
+
+# Re-export core exceptions for backward compatibility
+AppError = AppException
+
+T = TypeVar('T')
+
+
+# Exception classes are imported from app.core.exceptions above
+
+
+class ErrorHandler:
+    """
+    Centralized error handling system with consistent patterns.
+    """
+
+    @staticmethod
+    def handle_error(
+        error: Exception,
+        context: str = "",
+        show_user_message: bool = True,
+        log_error: bool = True,
+        recovery_hint: Optional[str] = None
+    ) -> None:
+        """
+        Handle an error with consistent logging and user messaging.
+
+        Args:
+            error: The exception that occurred
+            context: Description of where the error occurred
+            show_user_message: Whether to show error message to user
+            log_error: Whether to log the error
+            recovery_hint: Optional hint for user recovery
+        """
+        if log_error:
+            ErrorHandler._log_error(error, context)
+
+        if show_user_message:
+            ErrorHandler._show_user_error(error, recovery_hint)
+
+    @staticmethod
+    def _log_error(error: Exception, context: str = "") -> None:
+        """Log error with appropriate level based on error type"""
+        error_msg = f"{context}: {str(error)}" if context else str(error)
+
+        if isinstance(error, (ValidationError, ConfigError)):
+            logger.warning(error_msg)
+        elif isinstance(error, (ProcessingError, AIError)):
+            logger.error(error_msg)
+        else:
+            logger.exception(f"Unexpected error - {error_msg}")
+
+    @staticmethod
+    def _show_user_error(error: Exception, recovery_hint: Optional[str] = None) -> None:
+        """Show appropriate error message to user"""
+        from app.ui.ui_components import status_message
+
+        if isinstance(error, AppError):
+            user_message = error.user_message
+        else:
+            # For unexpected errors, don't show internal details
+            user_message = "An unexpected error occurred. Please try again."
+
+        # Add recovery hint if provided
+        if recovery_hint:
+            user_message += f"\n\n💡 {recovery_hint}"
+
+        # Show error message to user
+        if isinstance(error, ValidationError):
+            status_message(user_message, "warning")
+        else:
+            status_message(user_message, "error")
+
+    @staticmethod
+    def handle_with_recovery(
+        func: Callable[..., T],
+        context: str = "",
+        default_value: Any = None,
+        show_spinner: bool = False,
+        spinner_text: str = "Processing...",
+        recovery_hint: Optional[str] = None
+    ) -> Callable[..., T]:
+        """
+        Decorator that provides consistent error handling with recovery.
+
+        Args:
+            func: Function to wrap
+            context: Description of the operation
+            default_value: Value to return on error
+            show_spinner: Whether to show spinner during operation
+            spinner_text: Text to show in spinner
+            recovery_hint: Hint for user recovery
+
+        Returns:
+            Wrapped function with error handling
+        """
+        @wraps(func)
+        def wrapper(*args, **kwargs) -> T:
+            try:
+                if show_spinner:
+                    with st.spinner(spinner_text):
+                        return func(*args, **kwargs)
+                else:
+                    return func(*args, **kwargs)
+            except Exception as e:
+                ErrorHandler.handle_error(e, context, recovery_hint=recovery_hint)
+                return default_value
+
+        return wrapper
+
+    @staticmethod
+    def validate_input(value: Any, validator: Callable[[Any], bool], error_message: str) -> bool:
+        """
+        Validate input with consistent error handling.
+
+        Args:
+            value: Value to validate
+            validator: Function that returns True if valid
+            error_message: Error message if validation fails
+
+        Returns:
+            True if valid, False otherwise
+        """
+        try:
+            if validator(value):
+                return True
+            else:
+                raise ValidationError(error_message)
+        except ValidationError:
+            raise
+        except Exception as e:
+            raise ValidationError(f"Validation failed: {str(e)}")
+
+    @staticmethod
+    def ensure_config_value(config_value: Any, config_name: str) -> Any:
+        """
+        Ensure a configuration value exists and is valid.
+
+        Args:
+            config_value: The configuration value to check
+            config_name: Name of the configuration for error messages
+
+        Returns:
+            The config value if valid
+
+        Raises:
+            ConfigError: If config value is missing or invalid
+        """
+        if config_value is None or config_value == "":
+            raise ConfigError(
+                f"Configuration '{config_name}' is missing or empty",
+                user_message=f"Configuration error: {config_name} is not set",
+                recovery_hint="Please check your configuration and environment variables"
+            )
+        return config_value
+
+    @staticmethod
+    def handle_file_operation(
+        file_path: str,
+        operation: Callable[[], T],
+        operation_name: str = "file operation"
+    ) -> T:
+        """
+        Handle file operations with consistent error handling.
+
+        Args:
+            file_path: Path to the file being operated on
+            operation: Function that performs the file operation
+            operation_name: Description of the operation
+
+        Returns:
+            Result of the file operation
+        """
+        try:
+            return operation()
+        except FileNotFoundError:
+            raise ProcessingError(
+                f"File not found: {file_path}",
+                user_message=f"File not found: {file_path}",
+                recovery_hint="Please ensure the file exists and try again"
+            )
+        except PermissionError:
+            raise ProcessingError(
+                f"Permission denied accessing file: {file_path}",
+                user_message=f"Cannot access file: {file_path}",
+                recovery_hint="Please check file permissions"
+            )
+        except Exception as e:
+            raise ProcessingError(
+                f"Failed to {operation_name} file {file_path}: {str(e)}",
+                user_message=f"File operation failed: {operation_name}",
+                recovery_hint="Please check the file and try again"
+            )
+
+
+# Convenience decorators for common patterns
+def handle_ui_errors(context: str = "", recovery_hint: Optional[str] = None):
+    """
+    Decorator for UI operations that need error handling.
+
+    Args:
+        context: Description of the operation
+        recovery_hint: Optional hint for user recovery
+    """
+    def decorator(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            try:
+                return func(*args, **kwargs)
+            except Exception as e:
+                ErrorHandler.handle_error(e, context, recovery_hint=recovery_hint)
+                return None
+        return wrapper
+    return decorator
+
+
+def handle_processing_errors(context: str = "", recovery_hint: Optional[str] = None):
+    """
+    Decorator for processing operations that need error handling.
+
+    Args:
+        context: Description of the operation
+        recovery_hint: Optional hint for user recovery
+    """
+    def decorator(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            try:
+                return func(*args, **kwargs)
+            except Exception as e:
+                ErrorHandler.handle_error(e, context, recovery_hint=recovery_hint)
+                raise  # Re-raise for caller to handle
+        return wrapper
+    return decorator
+
+
+def validate_and_execute(
+    validator: Callable[[], bool],
+    operation: Callable[[], T],
+    validation_error_msg: str = "Validation failed",
+    context: str = ""
+) -> T:
+    """
+    Validate and execute operation with consistent error handling.
+
+    Args:
+        validator: Function that returns True if validation passes
+        operation: Function to execute if validation passes
+        validation_error_msg: Error message for validation failure
+        context: Description of the operation
+
+    Returns:
+        Result of the operation
+
+    Raises:
+        ValidationError: If validation fails
+    """
+    try:
+        if not validator():
+            raise ValidationError(validation_error_msg, recovery_hint="Please check your input and try again")
+        return operation()
+    except ValidationError:
+        raise
+    except Exception as e:
+        ErrorHandler.handle_error(e, f"{context} - validation/execution failed")
+        raise

app/ui/session_manager.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+"""
+Session State Manager
+
+Manages Streamlit session state with type-safe access.
+"""
+
+import streamlit as st
+from typing import Any
+
+from app.ui.error_handler import ErrorHandler
+
+
+
+class SessionProperty:
+    """
+    Descriptor for session state properties with type-safe access.
+
+    This descriptor provides a clean interface to Streamlit's session state,
+    eliminating repetitive property definitions while maintaining type safety.
+    """
+
+    def __init__(self, default_value: Any = None):
+        self.default_value = default_value
+        self.name = None
+
+    def __set_name__(self, owner, name):
+        self.name = name
+
+    def __get__(self, instance, owner):
+        if instance is None:
+            return self
+        return st.session_state.get(self.name, self.default_value)
+
+    def __set__(self, instance, value):
+        st.session_state[self.name] = value
+
+
+class SessionManager:
+    """Session state manager with type-safe access to session data."""
+
+    # Document processing state
+    documents = SessionProperty({})
+    chunks = SessionProperty([])
+    embeddings = SessionProperty(None)
+
+    # Analysis results
+    checklist_results = SessionProperty({})
+    question_answers = SessionProperty({})
+    overview_summary = SessionProperty("")
+    strategic_summary = SessionProperty("")
+
+    # User selections
+    strategy_path = SessionProperty(None)
+    strategy_text = SessionProperty("")
+    checklist_path = SessionProperty(None)
+    checklist_text = SessionProperty("")
+    questions_path = SessionProperty(None)
+    questions_text = SessionProperty("")
+    vdr_store = SessionProperty(None)
+    data_room_path = SessionProperty(None)
+
+    # Processing state
+    processing_active = SessionProperty(False)
+    agent = SessionProperty(None)
+
+    # Cached data
+    checklist = SessionProperty({})
+    questions = SessionProperty({})
+
+    def __init__(self) -> None:
+        """Initialize session state manager with default values."""
+        self._init_defaults()
+
+    def _init_defaults(self) -> None:
+        """Initialize default session state values."""
+        try:
+            # Get all descriptor properties and their defaults
+            all_properties = {
+                name: getattr(self.__class__, name).default_value
+                for name in dir(self.__class__)
+                if isinstance(getattr(self.__class__, name), SessionProperty)
+            }
+
+            for key, default_value in all_properties.items():
+                if key not in st.session_state:
+                    st.session_state[key] = default_value
+
+        except Exception as e:
+            ErrorHandler.handle_error(
+                e,
+                "Session initialization failed",
+                recovery_hint="Please refresh the page and try again"
+            )
+            # Initialize with minimal defaults on error
+            st.session_state.clear()
+            st.session_state.update({
+                'documents': {},
+                'processing_active': False,
+                'agent': None,
+            })
+
+
+    def reset(self) -> None:
+        """Reset analysis results and cached data for fresh analysis."""
+        self.overview_summary = ""
+        self.strategic_summary = ""
+        self.checklist_results = {}
+        self.question_answers = {}
+
+    def reset_processing(self) -> None:
+        """Reset processing flags to allow new operations."""
+        self.processing_active = False
+
+    def ready(self) -> bool:
+        """Check if system is ready for analysis operations."""
+        return bool(self.documents is not None and len(self.documents) > 0 and not self.processing_active)

app/ui/sidebar.py

@@ -0,0 +1,164 @@
+#!/usr/bin/env python3
+"""
+Sidebar Component
+
+Handles project selection, file selectors, and AI settings.
+"""
+
+import streamlit as st
+from pathlib import Path
+from typing import Tuple, Optional
+
+from app.ui.session_manager import SessionManager
+# Use lazy imports to avoid circular import issues
+# from app.handlers.document_handler import DocumentHandler
+# from app.handlers.ai_handler import AIHandler
+# Import components directly to avoid circular import issues
+import importlib.util
+import os
+
+# Load the ui_components.py module directly
+components_path = os.path.join(os.path.dirname(__file__), 'ui_components.py')
+spec = importlib.util.spec_from_file_location("components_module", components_path)
+components_module = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(components_module)
+
+# Import the specific functions we need
+render_project_selector = components_module.render_project_selector
+render_ai_settings = components_module.render_ai_settings
+render_file_selector = components_module.render_file_selector
+display_processing_error = components_module.display_processing_error
+status_message = components_module.status_message
+from app.core import logger
+
+
+class Sidebar:
+    """
+    Simplified sidebar component that handles all sidebar functionality.
+    """
+
+    def __init__(self, session: SessionManager, config):
+        """Initialize sidebar with session manager and config"""
+        self.session = session
+        self.config = config
+        # Handlers will be imported lazily when needed
+        self._document_handler = None
+        self._ai_handler = None
+
+    @property
+    def document_handler(self):
+        """Lazy import of DocumentHandler"""
+        if self._document_handler is None:
+            from app.handlers.document_handler import DocumentHandler
+            self._document_handler = DocumentHandler(self.session)
+        return self._document_handler
+
+    @property 
+    def ai_handler(self):
+        """Lazy import of AIHandler"""
+        if self._ai_handler is None:
+            from app.handlers.ai_handler import AIHandler
+            self._ai_handler = AIHandler(self.session)
+        return self._ai_handler
+
+    def render(self) -> Tuple[Optional[str], bool]:
+        """
+        Render sidebar with project selection, file selectors, and AI settings
+
+        Returns:
+            Tuple of (data_room_path, process_button_pressed)
+        """
+        with st.sidebar:
+            # Project and data room selection
+            selected_project_path, data_room_path = render_project_selector()
+
+            # Process button
+            process_button = st.button(
+                "🚀 Process Data Room",
+                type="primary",
+                width='stretch'
+            )
+
+            if process_button:
+                st.success("Processing... Check main area for progress")
+
+            st.divider()
+
+            # Analysis Configuration
+            st.subheader("📋 Analysis Configuration")
+
+            # Strategy selector
+            strategy_path, strategy_text = self._render_file_selector(
+                self.config.paths['strategy_dir'], "Strategy", "🎯"
+            )
+            self.session.strategy_path = strategy_path
+            self.session.strategy_text = strategy_text
+
+            # Checklist selector
+            checklist_path, checklist_text = self._render_file_selector(
+                self.config.paths['checklist_dir'], "Checklist", "📊"
+            )
+            self.session.checklist_path = checklist_path
+            self.session.checklist_text = checklist_text
+
+            # Questions selector
+            questions_path, questions_text = self._render_file_selector(
+                self.config.paths['questions_dir'], "Questions", "❓"
+            )
+            self.session.questions_path = questions_path
+            self.session.questions_text = questions_text
+
+            st.divider()
+
+            # AI settings
+            api_key, model_choice = render_ai_settings()
+
+            # Initialize AI agent if API key is available
+            if api_key:
+                existing_agent = self.session.agent
+                if existing_agent is None:
+                    if self.ai_handler.setup_agent(api_key, model_choice):
+                        st.success("✅ AI Agent ready")
+            else:
+                self.session.agent = None
+
+        return data_room_path, process_button
+
+    def _render_file_selector(self, directory: str, label: str, icon: str) -> Tuple[Optional[str], str]:
+        """
+        Render a file selector for a specific directory
+
+        Args:
+            directory: Path to the directory containing files
+            label: Label for the selector
+            icon: Icon for the selector
+
+        Returns:
+            Tuple of (selected_file_path, selected_file_content)
+        """
+        try:
+            return render_file_selector(directory, label, "sidebar", icon)
+        except Exception as e:
+            logger.error(f"Failed to render {label.lower()} selector: {e}")
+            return None, ""
+
+    def process_data_room(self, data_room_path: str):
+        """
+        Process a data room using the fast FAISS loading approach
+
+        Args:
+            data_room_path: Path to the data room directory
+        """
+        try:
+            result = self.document_handler.process_data_room_fast(data_room_path)
+
+            if result:
+                doc_count, chunk_count = result
+                st.success(f"✅ Loaded {doc_count} documents and {chunk_count} chunks from pre-built index!")
+                st.rerun()
+            else:
+                display_processing_error("data room")
+        except Exception as e:
+            logger.error(f"Failed to process data room {data_room_path}: {e}")
+            display_processing_error("data room", e)
+

app/ui/tabs/__init__.py

@@ -0,0 +1,21 @@
+"""
+Tab Components Package
+
+Contains all tab-specific UI components and logic.
+"""
+
+from .tab_base import TabBase
+from .overview_tab import OverviewTab
+from .strategic_tab import StrategicTab
+from .checklist_tab import ChecklistTab
+from .questions_tab import QuestionsTab
+from .qa_tab import QATab
+
+__all__ = [
+    'TabBase',
+    'OverviewTab',
+    'StrategicTab',
+    'ChecklistTab',
+    'QuestionsTab',
+    'QATab'
+]

app/ui/tabs/checklist_tab.py

@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+"""
+Checklist Tab Component
+
+Handles checklist matching and display.
+"""
+
+import streamlit as st
+
+from app.ui.session_manager import SessionManager
+from app.ui.ui_components import (
+    status_message,
+    render_generate_buttons,
+    processing_guard,
+    display_generation_error,
+    display_initialization_error
+)
+from app.handlers.ai_handler import AIHandler
+from app.core.logging import logger
+
+
+class ChecklistTab:
+    """
+    Checklist matching tab that handles checklist analysis and display.
+    """
+
+    def __init__(self, session: SessionManager, config, ai_handler: AIHandler):
+        """Initialize tab with session manager, config, and AI handler"""
+        self.session = session
+        self.config = config
+        self.ai_handler = ai_handler
+
+    def render(self):
+        """Render the checklist tab"""
+        documents = self.session.documents
+        if not documents:
+            status_message("👈 Configure and process data room first", "info")
+            return
+
+        # Use checklist from sidebar
+        file_text = self.session.checklist_text
+
+        if not file_text:
+            status_message("👈 Select a checklist in the sidebar first", "info")
+            return
+
+        # Generate button row
+        button_clicked = render_generate_buttons(
+            "📊 Generate Matching",
+            "regenerate_checklist_btn",
+            "checklist_results",
+            "Generate checklist matching analysis",
+            self.session
+        )
+
+        # Generate or display content
+        if button_clicked and not self.session.checklist_results:
+            self._generate_checklist_matching()
+        elif self.session.checklist_results:
+            from app.ui.ui_components import render_checklist_results
+            results = self.session.checklist_results
+            render_checklist_results(results, relevancy_threshold=self.config.processing['similarity_threshold'])
+        else:
+            status_message("👆 Click 'Generate Matching' to analyze checklist items against documents", "info")
+
+    @processing_guard()
+    def _generate_checklist_matching(self):
+        """Generate checklist matching analysis"""
+        # Initialize document processor with loaded FAISS store
+        from app.core import create_document_processor
+        
+        # Get the store name from session (set during data room processing)
+        store_name = self.session.vdr_store
+        if not store_name:
+            st.error("❌ No data room processed. Please process a data room first.")
+            return
+            
+        document_processor = create_document_processor(store_name=store_name)
+
+        try:
+            checklist_text = self.session.checklist_text
+            if not checklist_text or not self.session.chunks:
+                st.error("❌ No checklist or document chunks available")
+                return
+
+            # Check if data room has been processed
+            if not hasattr(self.session, 'documents') or not self.session.documents:
+                st.error("❌ No data room processed. Please process a data room first before running checklist analysis.")
+                return
+
+            # Note: Document type embeddings will be auto-loaded if missing during processing
+
+            with st.spinner("Processing checklist, please wait..."):
+                from app.core.parsers import parse_checklist
+                from app.core import search_and_analyze
+
+                try:
+                    # Parse raw checklist
+                    llm = self.ai_handler.llm
+                    if not llm:
+                        raise ValueError("AI service not configured. Please set up your API key first.")
+                    checklist = parse_checklist(checklist_text, llm)
+                    self.session.checklist = checklist
+
+                    # Use pre-built FAISS index from document processor
+                    if not document_processor.vector_store:
+                        raise ValueError("No pre-built FAISS index loaded. Please ensure data room is processed first.")
+
+                    vector_store = document_processor.vector_store
+
+                    # Process checklist items
+                    checklist_results = search_and_analyze(
+                        checklist,
+                        vector_store,
+                        self.ai_handler.session.agent.llm if self.ai_handler.is_agent_available() else None,
+                        self.config.processing['similarity_threshold'],
+                        'items',
+                        store_name=getattr(document_processor, 'store_name', None),
+                        session=self.session
+                    )
+                    self.session.checklist_results = checklist_results
+
+                    status_message("✅ Checklist matching analysis completed!", "success")
+                    st.rerun()
+
+                except Exception as e:
+                    logger.error(f"Checklist processing failed: {e}")
+                    display_generation_error("checklist analysis", e)
+
+        except Exception as e:
+            logger.error(f"Failed to initialize document processor: {e}")
+            display_initialization_error("document processor", e)
+        finally:
+            # Processing state is managed by processing_guard decorator
+            pass
+

app/ui/tabs/graph_tab.py

@@ -0,0 +1,548 @@
+#!/usr/bin/env python3
+"""
+Knowledge Graph Tab
+
+This tab provides an interface for exploring pre-computed knowledge graphs
+generated from due diligence documents. It offers entity search, relationship
+exploration, and graph analysis capabilities.
+"""
+
+import streamlit as st
+import plotly.graph_objects as go
+import plotly.express as px
+import pandas as pd
+from typing import Dict, List, Any, Optional
+
+from app.core.knowledge_graph import KnowledgeGraphManager, get_available_knowledge_graphs
+from app.ui.tabs.tab_base import TabBase
+from app.ui.error_handler import handle_ui_errors
+from app.core.logging import logger
+
+class GraphTab(TabBase):
+    """Knowledge Graph exploration tab"""
+    
+    def __init__(self, session_manager, config, ai_handler, export_handler):
+        super().__init__(session_manager, config, ai_handler, export_handler)
+        self.tab_name = "Knowledge Graph"
+        self.tab_key = "graph"
+    
+    @handle_ui_errors("Knowledge Graph", "Please try refreshing the page")
+    def render(self):
+        """Render the knowledge graph tab"""
+        st.header("🧠 Knowledge Graph Explorer")
+        
+        # Check if we have a loaded company
+        if not self.session.vdr_store:
+            st.info("📋 Please load a company first using the sidebar.")
+            return
+        
+        company_name = self.session.vdr_store
+        
+        # Initialize knowledge graph manager
+        if f'kg_manager_{company_name}' not in st.session_state:
+            st.session_state[f'kg_manager_{company_name}'] = KnowledgeGraphManager(company_name)
+        
+        kg_manager = st.session_state[f'kg_manager_{company_name}']
+        
+        # Load graph if not already loaded
+        if not kg_manager.is_available():
+            with st.spinner("Loading knowledge graph..."):
+                if not kg_manager.load_graph():
+                    st.error("❌ Knowledge graph not found for this company.")
+                    st.info("💡 Run `python scripts/build_knowledge_graphs.py` to generate knowledge graphs.")
+                    return
+        
+        # Display graph summary
+        self._render_graph_summary(kg_manager)
+        
+        # Main interface tabs
+        tab1, tab2, tab3, tab4, tab5 = st.tabs([
+            "🔍 Entity Search", 
+            "🔗 Relationship Explorer", 
+            "📊 Graph Analysis",
+            "🎯 Path Finder",
+            "🧠 Semantic Search"
+        ])
+        
+        with tab1:
+            self._render_entity_search(kg_manager)
+        
+        with tab2:
+            self._render_relationship_explorer(kg_manager)
+        
+        with tab3:
+            self._render_graph_analysis(kg_manager)
+        
+        with tab4:
+            self._render_path_finder(kg_manager)
+        
+        with tab5:
+            self._render_semantic_search(kg_manager)
+    
+    def _render_graph_summary(self, kg_manager: KnowledgeGraphManager):
+        """Render graph summary statistics"""
+        stats = kg_manager.get_summary_stats()
+        
+        if not stats:
+            return
+        
+        # Summary metrics
+        col1, col2, col3, col4 = st.columns(4)
+        
+        with col1:
+            st.metric("Total Entities", stats.get('num_entities', 0))
+        
+        with col2:
+            st.metric("Relationships", stats.get('num_relationships', 0))
+        
+        with col3:
+            entity_types = stats.get('entity_types', {})
+            st.metric("Entity Types", len(entity_types))
+        
+        with col4:
+            rel_types = stats.get('relationship_types', {})
+            st.metric("Relationship Types", len(rel_types))
+        
+        # Entity distribution chart
+        if entity_types:
+            with st.expander("📊 Entity Distribution", expanded=False):
+                fig = px.pie(
+                    values=list(entity_types.values()),
+                    names=list(entity_types.keys()),
+                    title="Distribution of Entity Types"
+                )
+                st.plotly_chart(fig, width='stretch')
+    
+    def _render_entity_search(self, kg_manager: KnowledgeGraphManager):
+        """Render entity search interface"""
+        st.subheader("🔍 Search Entities")
+        
+        # Search controls
+        col1, col2 = st.columns([3, 1])
+        
+        with col1:
+            search_query = st.text_input(
+                "Search for entities (companies, people, contracts, etc.)",
+                placeholder="e.g., Microsoft, John Smith, acquisition...",
+                key="entity_search_query"
+            )
+        
+        with col2:
+            entity_types = ['All'] + list(kg_manager.get_summary_stats().get('entity_types', {}).keys())
+            selected_type = st.selectbox(
+                "Filter by type",
+                entity_types,
+                key="entity_type_filter"
+            )
+        
+        if search_query:
+            # Perform search
+            filter_type = None if selected_type == 'All' else selected_type
+            results = kg_manager.search_entities(
+                search_query, 
+                entity_type=filter_type,
+                limit=20
+            )
+            
+            if results:
+                st.success(f"Found {len(results)} matching entities")
+                
+                # Display results
+                for i, entity in enumerate(results):
+                    with st.expander(f"🏷️ {entity['name']} ({entity['type']})", expanded=i==0):
+                        col1, col2 = st.columns([2, 1])
+                        
+                        with col1:
+                            st.write(f"**Type:** {entity['type']}")
+                            st.write(f"**Sources:** {entity['sources']}")
+                            st.write(f"**Document Type:** {entity['document_type']}")
+                            
+                            # Show context samples
+                            if entity.get('context_samples'):
+                                st.write("**Context:**")
+                                for context in entity['context_samples']:
+                                    if context.strip():
+                                        st.write(f"_{context.strip()}_")
+                        
+                        with col2:
+                            st.metric("Relevance Score", f"{entity['score']:.2f}")
+                            
+                            # Button to explore relationships
+                            if st.button(f"Explore Relationships", key=f"explore_{i}"):
+                                st.session_state['selected_entity'] = entity['name']
+                                st.rerun()
+            else:
+                st.info("No entities found matching your search criteria.")
+    
+    def _render_relationship_explorer(self, kg_manager: KnowledgeGraphManager):
+        """Render relationship exploration interface"""
+        st.subheader("🔗 Relationship Explorer")
+        
+        # Entity selection
+        selected_entity = st.session_state.get('selected_entity', '')
+        entity_input = st.text_input(
+            "Enter entity name to explore relationships",
+            value=selected_entity,
+            placeholder="e.g., Microsoft, John Smith...",
+            key="relationship_entity_input"
+        )
+        
+        if entity_input:
+            # Get relationships
+            relationships = kg_manager.get_entity_relationships(entity_input)
+            
+            if relationships['outgoing'] or relationships['incoming']:
+                # Display outgoing relationships
+                if relationships['outgoing']:
+                    st.write("### ➡️ Outgoing Relationships")
+                    outgoing_data = []
+                    for rel in relationships['outgoing']:
+                        outgoing_data.append({
+                            'Target': rel['target'],
+                            'Type': rel['target_type'],
+                            'Relationship': rel['relationship'],
+                            'Source Doc': rel['source_document'],
+                            'Confidence': f"{rel['confidence']:.2f}"
+                        })
+                    
+                    df_out = pd.DataFrame(outgoing_data)
+                    st.dataframe(df_out, width='stretch')
+                    
+                    # Show relationship context on selection
+                    if st.checkbox("Show relationship contexts", key="show_outgoing_context"):
+                        for i, rel in enumerate(relationships['outgoing']):
+                            if rel['context'].strip():
+                                st.write(f"**{rel['target']} ({rel['relationship']}):**")
+                                st.write(f"_{rel['context']}_")
+                                st.write("---")
+                
+                # Display incoming relationships
+                if relationships['incoming']:
+                    st.write("### ⬅️ Incoming Relationships")
+                    incoming_data = []
+                    for rel in relationships['incoming']:
+                        incoming_data.append({
+                            'Source': rel['source'],
+                            'Type': rel['source_type'],
+                            'Relationship': rel['relationship'],
+                            'Source Doc': rel['source_document'],
+                            'Confidence': f"{rel['confidence']:.2f}"
+                        })
+                    
+                    df_in = pd.DataFrame(incoming_data)
+                    st.dataframe(df_in, width='stretch')
+                    
+                    # Show relationship context on selection
+                    if st.checkbox("Show relationship contexts", key="show_incoming_context"):
+                        for i, rel in enumerate(relationships['incoming']):
+                            if rel['context'].strip():
+                                st.write(f"**{rel['source']} ({rel['relationship']}):**")
+                                st.write(f"_{rel['context']}_")
+                                st.write("---")
+                
+                # Relationship type distribution
+                all_rels = relationships['outgoing'] + relationships['incoming']
+                rel_types = {}
+                for rel in all_rels:
+                    rel_type = rel['relationship']
+                    rel_types[rel_type] = rel_types.get(rel_type, 0) + 1
+                
+                if rel_types:
+                    st.write("### 📊 Relationship Type Distribution")
+                    fig = px.bar(
+                        x=list(rel_types.keys()),
+                        y=list(rel_types.values()),
+                        title=f"Relationships for {entity_input}"
+                    )
+                    st.plotly_chart(fig, width='stretch')
+            
+            else:
+                st.info(f"No relationships found for '{entity_input}'. Try a different entity name.")
+    
+    def _render_graph_analysis(self, kg_manager: KnowledgeGraphManager):
+        """Render graph analysis interface"""
+        st.subheader("📊 Graph Analysis")
+        
+        # Central entities
+        st.write("### 🎯 Most Important Entities")
+        central_entities = kg_manager.get_central_entities(limit=15)
+        
+        if central_entities:
+            # Create a bar chart of centrality scores
+            names = [e['name'] for e in central_entities]
+            scores = [e['centrality_score'] for e in central_entities]
+            types = [e['type'] for e in central_entities]
+            
+            fig = px.bar(
+                x=scores,
+                y=names,
+                orientation='h',
+                color=types,
+                title="Entity Centrality Scores",
+                labels={'x': 'Centrality Score', 'y': 'Entity'}
+            )
+            fig.update_layout(height=500)
+            st.plotly_chart(fig, width='stretch')
+            
+            # Display detailed table
+            with st.expander("📋 Detailed Central Entities", expanded=False):
+                central_df = pd.DataFrame([{
+                    'Entity': e['name'],
+                    'Type': e['type'],
+                    'Centrality Score': e['centrality_score'],
+                    'Connections': e['num_connections'],
+                    'Sources': e['sources']
+                } for e in central_entities])
+                st.dataframe(central_df, width='stretch')
+        
+        # Entity clusters
+        st.write("### 🎭 Entity Clusters")
+        clusters = kg_manager.get_entity_clusters()
+        
+        if clusters:
+            st.info(f"Found {len(clusters)} clusters of related entities")
+            
+            for i, cluster in enumerate(clusters):
+                with st.expander(f"Cluster {i+1} ({len(cluster)} entities)", expanded=i==0):
+                    # Display cluster as tags
+                    cluster_html = " • ".join([f"**{entity}**" for entity in cluster])
+                    st.write(cluster_html)
+        else:
+            st.info("No significant entity clusters found.")
+    
+    def _render_path_finder(self, kg_manager: KnowledgeGraphManager):
+        """Render path finding interface"""
+        st.subheader("🎯 Path Finder")
+        st.write("Find connections between two entities in the knowledge graph.")
+        
+        col1, col2 = st.columns(2)
+        
+        with col1:
+            source_entity = st.text_input(
+                "Source Entity",
+                placeholder="e.g., Microsoft",
+                key="path_source_entity"
+            )
+        
+        with col2:
+            target_entity = st.text_input(
+                "Target Entity",
+                placeholder="e.g., OpenAI",
+                key="path_target_entity"
+            )
+        
+        max_length = st.slider("Maximum Path Length", 1, 5, 3, key="max_path_length")
+        
+        if source_entity and target_entity and st.button("Find Paths", key="find_paths_btn"):
+            with st.spinner("Searching for paths..."):
+                paths = kg_manager.find_paths(source_entity, target_entity, max_length)
+            
+            if paths:
+                st.success(f"Found {len(paths)} path(s) between {source_entity} and {target_entity}")
+                
+                for i, path in enumerate(paths):
+                    st.write(f"**Path {i+1}:**")
+                    path_str = " → ".join(path)
+                    st.write(f"🔗 {path_str}")
+                    
+                    # Show path length
+                    st.write(f"_Length: {len(path)-1} steps_")
+                    st.write("---")
+            else:
+                st.info(f"No paths found between {source_entity} and {target_entity} within {max_length} steps.")
+        
+        # Path finding tips
+        with st.expander("💡 Path Finding Tips", expanded=False):
+            st.write("""
+            - **Entity names**: Use exact or partial entity names as they appear in the documents
+            - **Path length**: Shorter paths show direct connections, longer paths reveal indirect relationships
+            - **Multiple paths**: Different paths can reveal different types of business relationships
+            - **Use cases**: 
+                - Find how two companies are connected
+                - Trace investment or acquisition chains
+                - Discover business partnerships and alliances
+            """)
+    
+    def _render_semantic_search(self, kg_manager: KnowledgeGraphManager):
+        """Render semantic search interface using FAISS embeddings"""
+        st.subheader("🧠 Semantic Search")
+        st.write("Search entities using natural language queries powered by your existing FAISS embeddings.")
+        
+        # Semantic entity search
+        st.write("### 🔍 Semantic Entity Search")
+        semantic_query = st.text_input(
+            "Describe what you're looking for (e.g., 'technology companies', 'financial partnerships', 'recent acquisitions')",
+            placeholder="e.g., companies involved in AI partnerships",
+            key="semantic_entity_query"
+        )
+        
+        col1, col2 = st.columns([1, 1])
+        with col1:
+            semantic_limit = st.slider("Max results", 5, 20, 10, key="semantic_limit")
+        with col2:
+            similarity_threshold = st.slider("Similarity threshold", 0.1, 0.8, 0.3, key="similarity_threshold")
+        
+        if semantic_query and st.button("🔍 Semantic Search", key="semantic_search_btn"):
+            with st.spinner("Searching using AI embeddings..."):
+                results = kg_manager.semantic_search_entities(
+                    semantic_query, 
+                    limit=semantic_limit, 
+                    similarity_threshold=similarity_threshold
+                )
+            
+            if results:
+                st.success(f"Found {len(results)} semantically relevant entities")
+                
+                for i, entity in enumerate(results):
+                    with st.expander(f"🏷️ {entity['name']} ({entity['type']}) - Score: {entity['similarity_score']:.3f}", expanded=i==0):
+                        col1, col2 = st.columns([2, 1])
+                        
+                        with col1:
+                            st.write(f"**Type:** {entity['type']}")
+                            st.write(f"**Sources:** {entity['sources']}")
+                            st.write(f"**Document Type:** {entity['document_type']}")
+                            
+                            # Show matching context
+                            if entity.get('matching_context'):
+                                st.write("**Relevant Context:**")
+                                st.write(f"_{entity['matching_context']}_")
+                            
+                            # Show original context samples
+                            if entity.get('context_samples'):
+                                st.write("**Entity Context:**")
+                                for context in entity['context_samples']:
+                                    if context.strip():
+                                        st.write(f"_{context.strip()}_")
+                        
+                        with col2:
+                            st.metric("Similarity Score", f"{entity['similarity_score']:.3f}")
+                            
+                            # Button to explore relationships
+                            if st.button(f"Explore Relations", key=f"semantic_explore_{i}"):
+                                st.session_state['selected_entity'] = entity['name']
+                                st.rerun()
+            else:
+                st.info("No entities found matching your semantic query. Try adjusting the similarity threshold or rephrasing your query.")
+        
+        # Context-based related entities
+        st.write("### 🔗 Find Related by Context")
+        st.write("Find entities that appear in similar contexts to a reference entity.")
+        
+        context_entity = st.text_input(
+            "Reference entity name",
+            placeholder="e.g., Microsoft",
+            key="context_reference_entity"
+        )
+        
+        context_limit = st.slider("Max related entities", 3, 15, 5, key="context_limit")
+        
+        if context_entity and st.button("Find Related by Context", key="find_context_related_btn"):
+            with st.spinner("Finding contextually related entities..."):
+                related = kg_manager.find_related_entities_by_context(context_entity, limit=context_limit)
+            
+            if related:
+                st.success(f"Found {len(related)} contextually related entities")
+                
+                related_data = []
+                for entity in related:
+                    related_data.append({
+                        'Entity': entity['name'],
+                        'Type': entity['type'],
+                        'Similarity': f"{entity['similarity_score']:.3f}",
+                        'Reason': entity['relationship_reason'],
+                        'Sources': entity['sources']
+                    })
+                
+                df_related = pd.DataFrame(related_data)
+                st.dataframe(df_related, width='stretch')
+                
+                # Show context samples for selected entities
+                if st.checkbox("Show context samples", key="show_related_contexts"):
+                    for entity in related:
+                        if entity.get('context_samples'):
+                            st.write(f"**{entity['name']}:**")
+                            for context in entity['context_samples']:
+                                if context.strip():
+                                    st.write(f"_{context.strip()}_")
+                            st.write("---")
+            else:
+                st.info(f"No contextually related entities found for '{context_entity}'.")
+        
+        # Semantic path search
+        st.write("### 🎯 Semantic Path Discovery")
+        st.write("Find connection paths that are semantically relevant to your query.")
+        
+        path_query = st.text_input(
+            "Describe the type of connections you want to find",
+            placeholder="e.g., investment relationships, technology partnerships",
+            key="semantic_path_query"
+        )
+        
+        max_semantic_paths = st.slider("Max paths", 3, 10, 5, key="max_semantic_paths")
+        
+        if path_query and st.button("Find Semantic Paths", key="semantic_paths_btn"):
+            with st.spinner("Discovering relevant connection paths..."):
+                paths = kg_manager.semantic_path_search(path_query, max_paths=max_semantic_paths)
+            
+            if paths:
+                st.success(f"Found {len(paths)} relevant connection paths")
+                
+                for i, path_info in enumerate(paths):
+                    st.write(f"**Path {i+1}:** (Relevance: {path_info['relevance_score']:.3f})")
+                    path_str = " → ".join(path_info['path'])
+                    st.write(f"🔗 {path_str}")
+                    st.write(f"_{path_info['query_relevance']}_")
+                    st.write(f"Length: {path_info['path_length']} steps")
+                    st.write("---")
+            else:
+                st.info(f"No semantically relevant paths found for '{path_query}'.")
+        
+        # Semantic search tips
+        with st.expander("💡 Semantic Search Tips", expanded=False):
+            st.write("""
+            **Semantic Search Benefits:**
+            - Uses your existing FAISS embeddings for intelligent matching
+            - Finds entities based on meaning, not just keywords
+            - Discovers hidden relationships through context similarity
+            - Leverages the same AI models used in your document analysis
+            
+            **Query Examples:**
+            - "technology companies with AI focus"
+            - "recent merger and acquisition activity"
+            - "financial services partnerships"
+            - "regulatory compliance issues"
+            - "key executive leadership"
+            
+            **How it works:**
+            1. Your query is embedded using the same model as your documents
+            2. FAISS finds the most similar document chunks
+            3. Entities from those chunks are returned with similarity scores
+            4. Results are ranked by semantic relevance
+            
+            **Performance Notes:**
+            - Requires existing FAISS indices (same as your document search)
+            - No additional models or external services needed
+            - Leverages your pre-computed embeddings for fast results
+            """)
+
+    def get_status(self) -> Dict[str, Any]:
+        """Get current status of the knowledge graph tab"""
+        if not self.session.vdr_store:
+            return {
+                'ready': False,
+                'message': 'No company loaded'
+            }
+        
+        company_name = self.session.vdr_store
+        available_graphs = get_available_knowledge_graphs()
+        
+        if company_name not in available_graphs:
+            return {
+                'ready': False,
+                'message': f'Knowledge graph not available for {company_name}'
+            }
+        
+        return {
+            'ready': True,
+            'message': f'Knowledge graph ready for {company_name}'
+        }

app/ui/tabs/overview_tab.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+"""
+Overview Tab Component
+
+Handles company overview generation and display.
+"""
+
+# Standard library imports
+from pathlib import Path
+
+# Third-party imports
+import streamlit as st
+
+# Local imports
+from app.ui.tabs.tab_base import TabBase
+from app.ui.ui_components import status_message
+
+
+class OverviewTab(TabBase):
+    """
+    Company overview tab that handles overview generation and display.
+    """
+
+    def render(self):
+        """Render the overview tab"""
+        if not self._check_documents_available():
+            return
+
+        # Generate button row
+        button_clicked = self._render_generate_buttons(
+            "🤖 Generate Overview",
+            "regenerate_overview_btn",
+            "overview_summary",
+            "Use AI to generate company overview analysis"
+        )
+
+        # Generate or display content
+        if self._should_generate_content(button_clicked, "overview_summary"):
+            self._generate_report("overview", "overview_summary", "✅ Company overview generated successfully!")
+        else:
+            self._render_content_or_placeholder(
+                "overview_summary",
+                "👆 Click 'Generate Overview' to create AI-powered company analysis"
+            )
+
+    def _generate_report(self, report_type: str, session_attr: str, success_message: str):
+        """Generate company overview report using AI"""
+        if not self._check_ai_availability():
+            return
+
+        with st.spinner("Agent running, please wait..."):
+            data_room_name = self._get_data_room_name()
+
+            overview_summary = self.ai_handler.generate_report(
+                report_type,
+                documents=self.session.documents,
+                data_room_name=data_room_name,
+                strategy_text=self.session.strategy_text,
+                checklist_results=self.session.checklist_results
+            )
+
+            if overview_summary:
+                setattr(self.session, session_attr, overview_summary)
+                status_message(success_message, "success")
+                st.rerun()
+            else:
+                status_message("Failed to generate overview. Please try again.", "error")
+
+    def _get_export_method_name(self) -> str:
+        """Get export method name for overview reports"""
+        return "export_overview_report"
+
+    def _get_download_key(self) -> str:
+        """Get download button key for overview reports"""
+        return "export_overview_btn"
+

app/ui/tabs/qa_tab.py

@@ -0,0 +1,216 @@
+#!/usr/bin/env python3
+"""
+Q&A Tab Component
+
+Handles Q&A with citations functionality.
+"""
+
+# Standard library imports
+from pathlib import Path
+
+# Third-party imports
+import streamlit as st
+
+# Local imports
+from app.core import RELEVANCY_THRESHOLD, logger
+from app.handlers.ai_handler import AIHandler
+from app.ui.session_manager import SessionManager
+from app.ui.ui_components import (
+    display_processing_error,
+    display_generation_error,
+    display_download_error,
+    status_message
+)
+
+
+class QATab:
+    """
+    Q&A with citations tab that handles question answering and citation display.
+    """
+
+    def __init__(self, session: SessionManager, config, ai_handler: AIHandler):
+        """Initialize tab with session manager, config, and AI handler"""
+        self.session = session
+        self.config = config
+        self.ai_handler = ai_handler
+
+    def render(self):
+        """Render the Q&A tab"""
+        chunks = self.session.chunks
+        if not chunks:
+            status_message("👈 Process data room first to enable Q&A", "info")
+            return
+
+        # Question input
+        question = st.text_input(
+            "Ask a question about your documents:",
+            placeholder="e.g., What are the main risks? What is the revenue model? Who are the key customers?",
+            key="qa_question_input"
+        )
+
+        # Handle Q&A query if there's a question
+        if question:
+            st.divider()
+            self._handle_qa_query(question)
+
+    def _handle_qa_query(self, question: str):
+        """Handle Q&A query and display results"""
+        # Create a unique key for this Q&A session to prevent resets
+        qa_key = f"qa_results_{hash(question) % 100000}"
+        
+        # Check if we already have results for this question in session state
+        if qa_key not in st.session_state:
+            try:
+                from app.core import search_documents
+
+                # Initialize document processor with loaded FAISS store
+                from app.core import create_document_processor
+                
+                # Get the store name from session (set during data room processing)
+                store_name = self.session.vdr_store
+                if not store_name:
+                    st.error("❌ No data room processed. Please process a data room first.")
+                    return
+                    
+                document_processor = create_document_processor(store_name=store_name)
+
+                # Use lower threshold for Q&A to get more relevant results
+                qa_threshold = 0.15  # Lower threshold for QA to find more results
+
+                with st.spinner("🔍 Searching documents..."):
+                    results = search_documents(
+                        question,
+                        document_processor,
+                        top_k=self.config.ui['top_k_search_results'],
+                        threshold=qa_threshold
+                    )
+
+                    # Fallback: try with lower threshold if no results found
+                    if not results:
+                        logger.info(f"No results found with threshold {qa_threshold}, trying lower threshold...")
+                        fallback_threshold = 0.05  # Very low threshold as last resort
+                        results = search_documents(
+                            question,
+                            document_processor,
+                            top_k=self.config.ui['top_k_search_results'],
+                            threshold=fallback_threshold
+                        )
+                        if results:
+                            st.info(f"ℹ️ Found results with lower relevance threshold ({fallback_threshold})")
+
+                # Store results in session state to prevent resets
+                st.session_state[qa_key] = {
+                    'question': question,
+                    'results': results,
+                    'has_ai': self.ai_handler.is_agent_available()
+                }
+
+            except Exception as e:
+                logger.error(f"Failed to handle Q&A query: {e}")
+                display_processing_error("question", e)
+                return
+
+        # Render results from session state
+        qa_data = st.session_state[qa_key]
+        results = qa_data['results']
+        
+        if results:
+            # Use agent to synthesize answer if available
+            if qa_data['has_ai']:
+                self._render_ai_answer(question, results)
+            else:
+                self._render_direct_results(results)
+        else:
+            status_message("No relevant information found for your question.", "warning")
+
+    def _render_ai_answer(self, question: str, results: list):
+        """Render AI-generated answer with citations"""
+        st.markdown("### 🤖 AI Service Answer")
+        with st.spinner("AI processing, please wait..."):
+            try:
+                # Convert results to document format for context
+                context_docs = [f"From {r.get('source', 'Unknown')}:\n{r.get('text', '')}" for r in results[:3]]
+
+                # Use the AI handler
+                answer_text = self.ai_handler.answer_question(question, context_docs)
+
+                st.markdown(answer_text)
+
+            except Exception as e:
+                logger.error(f"Failed to generate AI answer: {e}")
+                display_generation_error("AI answer")
+
+        st.divider()
+        self._render_source_documents(results, question)
+
+    def _render_direct_results(self, results: list):
+        """Render direct search results without AI synthesis"""
+        st.markdown("### 📚 Relevant Documents")
+        self._render_source_documents(results)
+
+    def _render_source_documents(self, results: list, question: str = ""):
+        """Render source documents with download buttons"""
+        st.markdown("### 📚 Source Documents")
+
+        # Display source documents with download buttons
+        for i, result in enumerate(results[:3], 1):
+            with st.container():
+                col1, col2 = st.columns([5, 1])
+                with col1:
+                    text_content = result.get('text', '')
+                    excerpt = text_content[:200] + "..." if len(text_content) > 200 else text_content
+                    st.markdown(f"{i}. \"{excerpt}\")")
+
+                    # Create clickable link for the document
+                    doc_path = result.get('path', result.get('full_path', ''))
+                    doc_name = result.get('source', 'Unknown Document')
+                    doc_title = self._format_document_title(doc_name)
+
+                    # Show document info and citation
+                    doc_source = result.get('source', 'Unknown')
+                    citation = result.get('citation', '')
+                    st.caption(f"   📄 {doc_source} ({citation})" if citation else f"   📄 {doc_source}")
+
+                with col2:
+                    # Only show one download button
+                    self._render_qa_download_button(result, i, question)
+
+    def _format_document_title(self, doc_name: str) -> str:
+        """Format document title for display"""
+        try:
+            from app.core import format_document_title
+            return format_document_title(doc_name)
+        except Exception:
+            return doc_name
+
+    def _render_qa_download_button(self, result: dict, idx: int, question: str):
+        """Render download button for Q&A results"""
+        doc_path = result.get('path', '')
+        if doc_path:
+            # Create a more stable key that won't cause resets
+            doc_source = result.get('source', 'document')
+            button_key = f"qa_dl_{idx}_{hash(doc_path + question) % 100000}"
+
+            # Use consistent path resolution logic
+            try:
+                from app.ui.ui_components import _resolve_document_path
+                resolved_path = _resolve_document_path(doc_path)
+                
+                if resolved_path and resolved_path.exists():
+                    with open(resolved_path, 'rb') as f:
+                        file_bytes = f.read()
+
+                    st.download_button(
+                        label="📥 Download",
+                        data=file_bytes,
+                        file_name=resolved_path.name,  # Use actual filename
+                        mime="application/pdf",
+                        key=button_key,
+                        help=f"Download {doc_source}",
+                        width='stretch'
+                    )
+                else:
+                    st.caption("(unavailable)")
+            except Exception as e:
+                logger.error(f"Download failed: {str(e)}")
+                st.caption("(error)")

app/ui/tabs/questions_tab.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+"""
+Questions Tab Component
+
+Handles due diligence questions analysis and display.
+"""
+
+import streamlit as st
+
+from app.ui.session_manager import SessionManager
+from app.ui.ui_components import (
+    status_message,
+    render_generate_buttons,
+    processing_guard,
+    display_generation_error,
+    display_initialization_error
+)
+from app.handlers.ai_handler import AIHandler
+from app.core.logging import logger
+
+
+class QuestionsTab:
+    """
+    Questions tab that handles due diligence questions analysis and display.
+    """
+
+    def __init__(self, session: SessionManager, config, ai_handler: AIHandler):
+        """Initialize tab with session manager, config, and AI handler"""
+        self.session = session
+        self.config = config
+        self.ai_handler = ai_handler
+
+    def render(self):
+        """Render the questions tab"""
+        documents = self.session.documents
+        if not documents:
+            status_message("👈 Configure and process data room first", "info")
+            return
+
+        # Use questions from sidebar
+        file_text = self.session.questions_text
+
+        if not file_text:
+            status_message("👈 Select a questions list in the sidebar first", "info")
+            return
+
+        # Generate button row
+        button_clicked = render_generate_buttons(
+            "❓ Generate Answers",
+            "regenerate_questions_btn",
+            "question_answers",
+            "Generate answers for due diligence questions",
+            self.session
+        )
+
+        # Generate or display content
+        if button_clicked and not self.session.question_answers:
+            self._generate_question_answers()
+        elif self.session.question_answers:
+            from app.ui.ui_components import render_question_results
+            answers = self.session.question_answers
+            # Convert from {'questions': [...]} format to {question_id: answer_data} format
+            if isinstance(answers, dict) and 'questions' in answers:
+                questions_dict = {}
+                for i, question_data in enumerate(answers['questions']):
+                    questions_dict[f"question_{i}"] = question_data
+                render_question_results(questions_dict)
+            else:
+                render_question_results(answers)
+        else:
+            status_message("👆 Click 'Generate Answers' to find relevant documents for due diligence questions", "info")
+
+    @processing_guard()
+    def _generate_question_answers(self):
+        """Generate question answering analysis"""
+        from app.core.document_processor import DocumentProcessor
+
+        # Initialize document processor with loaded FAISS store
+        from app.core.utils import create_document_processor
+        
+        # Get the store name from session (set during data room processing)
+        store_name = self.session.vdr_store
+        if not store_name:
+            st.error("❌ No data room processed. Please process a data room first.")
+            return
+            
+        document_processor = create_document_processor(store_name=store_name)
+
+        try:
+            questions_text = self.session.questions_text
+            if not questions_text or not self.session.chunks:
+                st.error("❌ No questions or document chunks available")
+                return
+
+            # Show progress indicator
+            with st.spinner("🚀 Starting question analysis..."):
+                try:
+                    from app.core.parsers import parse_questions
+                    from app.core.search import search_and_analyze
+
+                    # Step 1: Parse questions
+                    st.info("📋 Parsing questions...")
+                    llm = self.ai_handler.llm
+                    if not llm:
+                        raise ValueError("AI service not configured. Please set up your API key first.")
+                    questions = parse_questions(questions_text, llm)
+                    self.session.questions = questions
+                    st.info(f"Found {len(questions)} questions to process")
+
+                    # Step 2: Use pre-built FAISS index
+                    st.info("🔍 Setting up document search...")
+                    if not document_processor.vector_store:
+                        raise ValueError("No pre-built FAISS index loaded. Please ensure data room is processed first.")
+                    vector_store = document_processor.vector_store
+
+                    # Step 3: Process questions with batch processing
+                    st.info("🤖 Processing questions with AI (batch mode)...")
+                    st.info("Using concurrent processing for faster results...")
+
+                    question_answers = search_and_analyze(
+                        questions,
+                        vector_store,
+                        self.ai_handler.session.agent.llm if self.ai_handler.is_agent_available() else None,
+                        self.config.processing['relevancy_threshold'],
+                        'questions',
+                        store_name=getattr(document_processor, 'store_name', None)
+                    )
+                    self.session.question_answers = question_answers
+
+                    # Complete
+                    questions_list = question_answers.get('questions', [])
+                    answered_count = sum(1 for a in questions_list if a.get('has_answer', False))
+                    st.success(f"✅ Completed! {answered_count}/{len(questions)} questions answered")
+
+                    status_message("✅ Question answering analysis completed!", "success")
+                    st.rerun()
+
+                except Exception as e:
+                    logger.error(f"Questions processing failed: {e}")
+                    display_generation_error("question analysis", e)
+        except Exception as e:
+            logger.error(f"Failed to initialize document processor: {e}")
+            display_initialization_error("document processor", e)

app/ui/tabs/strategic_tab.py

@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+"""
+Strategic Analysis Tab Component
+
+Handles strategic analysis generation and display.
+"""
+
+import streamlit as st
+
+from app.ui.tabs.tab_base import TabBase
+from app.ui.ui_components import status_message
+from app.core import logger
+
+
+class StrategicTab(TabBase):
+    """
+    Strategic analysis tab that handles strategic report generation and display.
+    """
+
+    def render(self):
+        """Render the strategic analysis tab"""
+        if not self._check_documents_available():
+            return
+
+        # Generate button row
+        button_clicked = self._render_generate_buttons(
+            "🎯 Generate Analysis",
+            "regenerate_strategic_btn",
+            "strategic_summary",
+            "Use AI to generate strategic analysis"
+        )
+
+        # Generate or display content
+        if self._should_generate_content(button_clicked, "strategic_summary"):
+            self._generate_report("strategic", "strategic_summary", "✅ Strategic analysis generated successfully!")
+        else:
+            self._render_content_or_placeholder(
+                "strategic_summary",
+                "👆 Click 'Generate Analysis' to create AI-powered strategic assessment"
+            )
+
+    def _generate_report(self, report_type: str, session_attr: str, success_message: str):
+        """Generate strategic analysis report using AI"""
+        if not self._check_ai_availability():
+            return
+
+        if not self._check_processing_active():
+            return
+
+        # Set processing active
+        self._set_processing_active(True)
+
+        try:
+            with st.spinner("Agent running, please wait..."):
+                data_room_name = self._get_data_room_name()
+
+                strategic_summary = self.ai_handler.generate_report(
+                    report_type,
+                    documents=self.session.documents,
+                    data_room_name=data_room_name,
+                    strategy_text=self.session.strategy_text,
+                    checklist_results=self.session.checklist_results
+                )
+
+                if strategic_summary:
+                    setattr(self.session, session_attr, strategic_summary)
+                    status_message(success_message, "success")
+                    st.rerun()
+                else:
+                    status_message("Failed to generate strategic analysis. Please try again.", "error")
+        except Exception as e:
+            logger.error(f"Failed to generate strategic analysis: {e}")
+            status_message(f"Failed to generate strategic analysis: {str(e)}", "error")
+        finally:
+            # Always reset processing state
+            self._set_processing_active(False)
+
+    def _get_export_method_name(self) -> str:
+        """Get export method name for strategic reports"""
+        return "export_strategic_report"
+
+    def _get_download_key(self) -> str:
+        """Get download button key for strategic reports"""
+        return "export_strategic_btn"
+

app/ui/tabs/tab_base.py

@@ -0,0 +1,141 @@
+#!/usr/bin/env python3
+"""
+Tab Base Component
+
+Provides shared functionality for all tab components including common
+initialization patterns, render methods, and export functionality.
+"""
+
+# Standard library imports
+from pathlib import Path
+from typing import Optional, Any, Dict
+
+# Third-party imports
+import streamlit as st
+
+# Local imports
+from app.ui.error_handler import handle_ui_errors
+from app.handlers.ai_handler import AIHandler
+from app.handlers.export_handler import ExportHandler
+from app.ui.session_manager import SessionManager
+from app.ui.ui_components import status_message, render_generate_buttons
+
+
+class TabBase:
+    """
+    Base class for tab components with shared functionality.
+
+    Provides common patterns for initialization, rendering, and export functionality.
+    """
+
+    def __init__(self, session: SessionManager, config, ai_handler: AIHandler, export_handler: ExportHandler):
+        """Initialize tab with session manager, config, and handlers"""
+        self.session = session
+        self.config = config
+        self.ai_handler = ai_handler
+        self.export_handler = export_handler
+
+    def render(self):
+        """Render the tab - to be implemented by subclasses"""
+        raise NotImplementedError("Subclasses must implement render()")
+
+    def _check_documents_available(self) -> bool:
+        """Check if documents are available and show message if not"""
+        if not self.session.documents:
+            status_message("👈 Configure and process data room first", "info")
+            return False
+        return True
+
+    def _render_generate_buttons(self, generate_label: str, regenerate_key: str,
+                               session_attr: str, help_text: str) -> tuple[bool, bool]:
+        """Render common generate and regenerate buttons using reusable component"""
+        return render_generate_buttons(
+            generate_label,
+            regenerate_key,
+            session_attr,
+            help_text,
+            self.session
+        )
+
+    def _should_generate_content(self, generate_clicked: bool, session_attr: str) -> bool:
+        """Determine if content should be generated"""
+        return generate_clicked and not getattr(self.session, session_attr)
+
+    def _should_display_content(self, session_attr: str) -> bool:
+        """Determine if content should be displayed"""
+        return bool(getattr(self.session, session_attr))
+
+    def _get_data_room_name(self) -> str:
+        """Get the data room name from documents"""
+        if not self.session.documents:
+            return "Unknown"
+        return Path(list(self.session.documents.keys())[0]).parent.name
+
+    def _check_ai_availability(self) -> bool:
+        """Check if AI agent is available"""
+        if not self.ai_handler.is_agent_available():
+            status_message("AI Agent not available. Please configure your API key in the sidebar.", "error")
+            return False
+        return True
+
+    def _check_processing_active(self) -> bool:
+        """Check if processing is already active"""
+        if self.session.processing_active:
+            status_message("⚠️ Another operation is currently running. Please wait.", "warning")
+            return False
+        return True
+
+    def _set_processing_active(self, active: bool):
+        """Set processing active state"""
+        self.session.processing_active = active
+
+    @handle_ui_errors("Report generation", "Please check your documents and try again")
+    def _generate_report(self, report_type: str, session_attr: str, success_message: str):
+        """Generate report using AI - to be implemented by subclasses"""
+        raise NotImplementedError("Subclasses must implement _generate_report()")
+
+    def _render_export_button(self, export_method_name: str, download_key: str):
+        """Render export button for reports"""
+        # Get the session attribute dynamically
+        session_attr = export_method_name.replace("export_", "").replace("_report", "_summary")
+        if not getattr(self.session, session_attr):
+            return
+
+        # Call the export method dynamically
+        export_method = getattr(self.export_handler, export_method_name)
+        file_name, export_data = export_method()
+
+        if file_name and export_data:
+            st.download_button(
+                "📥 Export Report",
+                data=export_data,
+                file_name=file_name,
+                mime="text/markdown",
+                key=download_key,
+                help="Download report as Markdown file"
+            )
+
+    def _render_content_or_placeholder(self, session_attr: str, placeholder_message: str):
+        """Render content if available, otherwise show placeholder"""
+        content = getattr(self.session, session_attr)
+        if content:
+            if isinstance(content, str):
+                st.markdown(content)
+            else:
+                # Handle dict/other types as needed by subclasses
+                self._render_custom_content(content)
+            self._render_export_button(self._get_export_method_name(), self._get_download_key())
+        else:
+            status_message(placeholder_message, "info")
+
+    def _render_custom_content(self, content: Any):
+        """Render custom content types - can be overridden by subclasses"""
+        pass
+
+    def _get_export_method_name(self) -> str:
+        """Get export method name - to be implemented by subclasses"""
+        raise NotImplementedError("Subclasses must implement _get_export_method_name()")
+
+    def _get_download_key(self) -> str:
+        """Get download button key - to be implemented by subclasses"""
+        raise NotImplementedError("Subclasses must implement _get_download_key()")