modularization and configuration

.env.example

@@ -0,0 +1,6 @@
+# BOQTenders Environment Configuration
+# Copy this file to .env and fill in your API keys
+
+# Required API Keys
+GOOGLE_API_KEY=your_google_api_key_here
+HF_API_TOKEN=your_huggingface_api_token_here

.gitignore

@@ -1,4 +1,5 @@
 __pycache__/
+**/__pycache__/
 *.pyc
 *.pyo
 *.pyd

api/routes.py

@@ -0,0 +1,252 @@
+"""
+FastAPI routes for BOQ extraction API.
+"""
+import tempfile
+from typing import Optional
+from pathlib import Path
+
+from fastapi import FastAPI, File, UploadFile, HTTPException
+from fastapi.middleware.cors import CORSMiddleware
+from loguru import logger
+
+from config.settings import settings
+from core.pdf_extractor import PDFExtractor
+from core.embeddings import EmbeddingService
+from core.rag_chain import RAGChainBuilder
+from services.boq_extractor import BOQExtractor
+from services.consistency import ConsistencyChecker
+from api.schemas import (
+    ChatRequest,
+    ChatResponse,
+    UploadResponse,
+    ConsistencyResponse,
+    ErrorResponse,
+)
+
+
+# Initialize FastAPI app
+app = FastAPI(
+    title=settings.api.title,
+    description=settings.api.description,
+    version=settings.api.version,
+    docs_url="/docs" if settings.api.docs_enabled else None,
+    redoc_url="/redoc" if settings.api.docs_enabled else None,
+)
+
+# Configure CORS
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=settings.api.cors_origins,
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# Global state for session data
+_session_state = {
+    "qa_chain": None,
+    "vector_store": None,
+    "chunks": None,
+}
+
+# Initialize services
+pdf_extractor = PDFExtractor()
+embedding_service = EmbeddingService()
+rag_builder = RAGChainBuilder()
+boq_extractor = BOQExtractor()
+consistency_checker = ConsistencyChecker(boq_extractor=boq_extractor)
+
+# Expose router for external use
+router = app.router
+
+
+@app.post(
+    "/upload",
+    response_model=UploadResponse,
+    responses={400: {"model": ErrorResponse}, 500: {"model": ErrorResponse}},
+    tags=["Documents"]
+)
+async def upload_pdf(file: UploadFile = File(...)):
+    """
+    Upload a PDF file for BOQ extraction.
+    
+    - Extracts text from PDF
+    - Creates embeddings and vector store
+    - Extracts BOQ items
+    - Sets up QA chain for chat
+    """
+    global _session_state
+    
+    if not file:
+        raise HTTPException(status_code=400, detail="No file uploaded")
+    
+    if not file.filename.lower().endswith('.pdf'):
+        raise HTTPException(status_code=400, detail="Only PDF files are supported")
+    
+    try:
+        logger.info(f'Processing uploaded file: {file.filename}')
+        
+        # Save to temp file
+        with tempfile.NamedTemporaryFile(delete=False, suffix='.pdf') as temp_file:
+            content = await file.read()
+            temp_file.write(content)
+            temp_path = temp_file.name
+        
+        try:
+            # Extract text from PDF
+            logger.info('Extracting text from PDF...')
+            text = pdf_extractor.extract_text(temp_path, filename=file.filename)
+            
+            if not text:
+                raise HTTPException(
+                    status_code=400,
+                    detail="Could not extract text from PDF"
+                )
+            
+            # Create chunks and vector store
+            logger.info('Creating embeddings...')
+            chunks = embedding_service.split_text(text)
+            vector_store = embedding_service.create_vector_store(chunks)
+            
+            # Extract BOQ
+            logger.info('Extracting BOQ...')
+            boq_output = boq_extractor.extract(chunks, vector_store)
+            
+            # Build QA chain
+            logger.info('Building QA chain...')
+            qa_chain = rag_builder.build(vector_store)
+            
+            # Store in session state
+            _session_state["qa_chain"] = qa_chain
+            _session_state["vector_store"] = vector_store
+            _session_state["chunks"] = chunks
+            
+            logger.info(f'Upload completed: {len(chunks)} chunks created')
+            
+            return UploadResponse(
+                message="success",
+                output=boq_output
+            )
+            
+        finally:
+            # Clean up temp file
+            Path(temp_path).unlink(missing_ok=True)
+            
+    except HTTPException:
+        raise
+    except Exception as e:
+        logger.error(f'Error processing upload: {e}')
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@app.post(
+    "/chat",
+    response_model=ChatResponse,
+    responses={400: {"model": ErrorResponse}, 500: {"model": ErrorResponse}},
+    tags=["Chat"]
+)
+async def chat(request: ChatRequest):
+    """
+    Ask a question about the uploaded document.
+    
+    Requires a document to be uploaded first via /upload endpoint.
+    """
+    global _session_state
+    
+    if not _session_state.get("qa_chain"):
+        raise HTTPException(
+            status_code=400,
+            detail="No document loaded. Please upload a PDF first."
+        )
+    
+    try:
+        logger.info(f'Processing chat question: {request.question[:50]}...')
+        
+        qa_chain = _session_state["qa_chain"]
+        
+        # Get response from QA chain (using old LangChain API)
+        response = qa_chain({"question": request.question})
+        
+        answer = response.get("answer", "")
+        
+        logger.info('Chat response generated')
+        
+        return ChatResponse(answer=answer)
+        
+    except Exception as e:
+        logger.error(f'Error processing chat: {e}')
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@app.post(
+    "/consistency",
+    response_model=ConsistencyResponse,
+    responses={400: {"model": ErrorResponse}, 500: {"model": ErrorResponse}},
+    tags=["Analysis"]
+)
+async def check_consistency(runs: int = 4):
+    """
+    Check extraction consistency by running multiple extractions.
+    
+    Requires a document to be uploaded first via /upload endpoint.
+    
+    Args:
+        runs: Number of extraction runs (default: 4)
+    """
+    global _session_state
+    
+    if not _session_state.get("chunks"):
+        raise HTTPException(
+            status_code=400,
+            detail="No document loaded. Please upload a PDF first."
+        )
+    
+    if runs < 2:
+        raise HTTPException(
+            status_code=400,
+            detail="At least 2 runs required for consistency check"
+        )
+    
+    if runs > 10:
+        raise HTTPException(
+            status_code=400,
+            detail="Maximum 10 runs allowed"
+        )
+    
+    try:
+        logger.info(f'Running consistency check with {runs} runs')
+        
+        result = consistency_checker.check(
+            chunks=_session_state["chunks"],
+            vector_store=_session_state["vector_store"],
+            runs=runs
+        )
+        
+        return ConsistencyResponse(
+            consistency_score=result.get("consistency_score"),
+            successful_runs=result.get("successful_runs"),
+            avg_confidence=result.get("avg_confidence"),
+            is_low_consistency=result.get("is_low_consistency")
+        )
+        
+    except Exception as e:
+        logger.error(f'Error in consistency check: {e}')
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@app.get(
+    "/clear",
+    tags=["Session"]
+)
+async def clear_session():
+    """Clear the current session state."""
+    global _session_state
+    
+    _session_state = {
+        "qa_chain": None,
+        "vector_store": None,
+        "chunks": None,
+    }
+    
+    logger.info('Session cleared')
+    return {"message": "Session cleared"}

api/schemas.py

@@ -0,0 +1,104 @@
+"""
+Pydantic schemas for API request/response validation.
+"""
+from typing import Optional, List, Dict, Any
+from pydantic import BaseModel, Field
+
+
+class ChatRequest(BaseModel):
+    """Chat endpoint request schema."""
+    question: str = Field(..., description="Question to ask about the document")
+    
+    class Config:
+        json_schema_extra = {
+            "example": {
+                "question": "What is the total quantity of steel required?"
+            }
+        }
+
+
+class ChatResponse(BaseModel):
+    """Chat endpoint response schema."""
+    answer: str = Field(..., description="Answer to the question")
+    
+    class Config:
+        json_schema_extra = {
+            "example": {
+                "answer": "The total quantity of steel required is 500 MT."
+            }
+        }
+
+
+class UploadResponse(BaseModel):
+    """Upload endpoint response schema."""
+    message: str = Field(..., description="Status message")
+    output: str = Field(..., description="Extracted BOQ in markdown format")
+    
+    class Config:
+        json_schema_extra = {
+            "example": {
+                "message": "success",
+                "output": "## DOCUMENT SUMMARY\n..."
+            }
+        }
+
+
+class ConsistencyResponse(BaseModel):
+    """Consistency check endpoint response schema."""
+    consistency_score: float = Field(
+        ...,
+        ge=0,
+        le=100,
+        description="Consistency score as percentage (0-100)"
+    )
+    successful_runs: int = Field(..., description="Number of successful runs")
+    avg_confidence: float = Field(
+        ...,
+        ge=0,
+        le=100,
+        description="Average confidence score"
+    )
+    is_low_consistency: bool = Field(
+        ...,
+        description="Whether consistency is below threshold"
+    )
+    
+    class Config:
+        json_schema_extra = {
+            "example": {
+                "consistency_score": 92.5,
+                "successful_runs": 4,
+                "avg_confidence": 85.2,
+                "is_low_consistency": False
+            }
+        }
+
+
+class BOQResponse(BaseModel):
+    """BOQ extraction response schema."""
+    boq_output: str = Field(..., description="Extracted BOQ in markdown format")
+    items_count: int = Field(default=0, description="Number of BOQ items extracted")
+    
+    class Config:
+        json_schema_extra = {
+            "example": {
+                "boq_output": "## DOCUMENT SUMMARY\n...",
+                "items_count": 25
+            }
+        }
+
+
+class ErrorResponse(BaseModel):
+    """Error response schema."""
+    error: str = Field(..., description="Error type")
+    message: str = Field(..., description="Error message")
+    detail: Optional[str] = Field(default=None, description="Additional error details")
+    
+    class Config:
+        json_schema_extra = {
+            "example": {
+                "error": "ValidationError",
+                "message": "No file uploaded",
+                "detail": "Please upload a PDF file"
+            }
+        }

app.py

@@ -1,122 +1,39 @@
-from fastapi import FastAPI, HTTPException, UploadFile, File
-from pydantic import BaseModel
-import boq_processor
-from loguru import logger
-import os
-import shutil
-from typing import Dict, Any
+"""
+BOQTenders API Server
 
-app = FastAPI(title="BOQ Chatbot API", description="API for extracting and querying BOQ from tender PDFs using RAG+CAG")
+FastAPI application entry point for BOQ extraction and chat services.
 
-# Global variables for chain and vector store
-vector_store = None
-qa_chain = None
-chunks = None
+Usage:
+    uvicorn app:app --host 0.0.0.0 --port 8000 --reload
+"""
+import sys
+from pathlib import Path
 
-class ChatRequest(BaseModel):
-    question: str
+# Add project root to path
+sys.path.insert(0, str(Path(__file__).parent))
 
-@app.post("/upload", summary="Upload PDF", description="Upload a PDF file to initialize the BOQ chatbot.")
-async def upload_pdf(file: UploadFile = File(...)) -> Dict[str, Any]:
-    global vector_store, qa_chain
-    if not file.filename.endswith(".pdf"):
-        logger.warning(f"Invalid file type uploaded: {file.filename}")
-        raise HTTPException(status_code=400, detail="Only PDF files are allowed")
-    
-    # Save uploaded file
-    pdf_path = f"temp_{file.filename}"
-    try:
-        with open(pdf_path, "wb") as buffer:
-            shutil.copyfileobj(file.file, buffer)
-        
-        logger.info(f"Processing uploaded PDF: {file.filename}")
-        global chunks
-        chunks = boq_processor.load_and_process_pdf(pdf_path, filename=file.filename)
-        vector_store = boq_processor.create_vector_store(chunks)
-        qa_chain = boq_processor.setup_rag_chain(vector_store)
-        
-        # Use comprehensive extraction for complete BOQ coverage
-        extracted_boq = boq_processor.extract_boq_comprehensive(chunks, vector_store)
-        
-        logger.info("PDF uploaded and processed successfully")
-        
-        # Return enriched response with processing metadata
-        return {
-            "status": "success",
-            "message": "PDF uploaded and processed successfully",
-            "file_name": file.filename,
-            "processing_info": {
-                "documents_loaded": len(chunks),
-                "chunks_created": len(chunks),
-                "vector_store_ready": vector_store is not None,
-                "rag_chain_ready": qa_chain is not None
-            },
-            "extracted_boq": extracted_boq
-        }
-    except Exception as e:
-        error_msg = str(e)
-        logger.error(f"Error processing PDF: {error_msg}")
-        
-        # Check if it's a rate limit error and provide specific guidance
-        if "429" in error_msg or "RESOURCE_EXHAUSTED" in error_msg or "quota" in error_msg.lower():
-            raise HTTPException(
-                status_code=429, 
-                detail="API rate limit exceeded. Please try again later or upgrade your API plan. See https://ai.google.dev/gemini-api/docs/rate-limits"
-            )
-        else:
-            raise HTTPException(status_code=500, detail="Internal server error")
-    finally:
-        if os.path.exists(pdf_path):
-            os.remove(pdf_path)
+from loguru import logger
+from config.settings import settings
+from api.routes import app
+
+# Configure logging
+logger.remove()
+logger.add(
+    sys.stderr,
+    level=settings.log_level,
+    format="<green>{time:YYYY-MM-DD HH:mm:ss}</green> | <level>{level: <8}</level> | <cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>"
+)
+
+# Export app for uvicorn
+__all__ = ["app"]
 
-@app.post("/chat", summary="Chat with BOQ", description="Send a question about the uploaded BOQ PDF and get an answer.")
-async def chat(request: ChatRequest) -> Dict[str, str]:
-    if not qa_chain:
-        logger.warning("Chat attempted without uploaded PDF")
-        raise HTTPException(status_code=400, detail="No PDF uploaded. Please upload a PDF first using /upload")
-    
-    try:
-        logger.info(f"Processing chat question: {request.question}")
-        
-        # Use old LangChain API (0.1.x) directly
-        result = qa_chain({"question": request.question})
-        
-        logger.info("Chat response generated")
-        return {"answer": result["answer"]}
-    except Exception as e:
-        error_msg = str(e)
-        logger.error(f"Error in chat: {error_msg}")
-        
-        # Check if it's a rate limit error and provide specific guidance
-        if "429" in error_msg or "RESOURCE_EXHAUSTED" in error_msg or "quota" in error_msg.lower():
-            raise HTTPException(
-                status_code=429, 
-                detail="API rate limit exceeded. Please try again later or upgrade your API plan. See https://ai.google.dev/gemini-api/docs/rate-limits"
-            )
-        else:
-            raise HTTPException(status_code=500, detail="Internal server error")
 
-@app.get("/consistency", summary="Check Consistency", description="Run multiple BOQ extractions to check LLM output consistency.")
-async def get_consistency() -> Dict[str, Any]:
-    global chunks, vector_store
-    if not chunks or not vector_store:
-        logger.warning("Consistency check attempted without uploaded PDF")
-        raise HTTPException(status_code=400, detail="No PDF uploaded. Please upload a PDF first using /upload")
+if __name__ == "__main__":
+    import uvicorn
     
-    try:
-        logger.info("Running consistency check")
-        consistency_result = boq_processor.check_consistency(chunks, vector_store, runs=4)
-        logger.info(f"Consistency check completed: {consistency_result}")
-        return {"status": "success", "consistency": consistency_result}
-    except Exception as e:
-        error_msg = str(e)
-        logger.error(f"Error in consistency check: {error_msg}")
-        
-        # Check if it's a rate limit error
-        if "429" in error_msg or "RESOURCE_EXHAUSTED" in error_msg or "quota" in error_msg.lower():
-            raise HTTPException(
-                status_code=429, 
-                detail="API rate limit exceeded. Please try again later or upgrade your API plan. See https://ai.google.dev/gemini-api/docs/rate-limits"
-            )
-        else:
-            raise HTTPException(status_code=500, detail="Internal server error")
+    uvicorn.run(
+        "app:app",
+        host=settings.api.host,
+        port=settings.api.port,
+        reload=settings.api.debug,
+    )

boq_processor.py

@@ -1,477 +0,0 @@
-import os
-from typing import List, Optional
-import dotenv
-from pydantic_settings import BaseSettings
-from loguru import logger
-from langchain_text_splitters import RecursiveCharacterTextSplitter
-from langchain_huggingface import HuggingFaceEmbeddings
-from langchain_community.vectorstores import FAISS
-from langchain_google_genai import GoogleGenerativeAI
-import time
-from functools import wraps
-import re
-import requests
-
-from langchain_core.documents import Document
-
-# Load environment variables
-dotenv.load_dotenv()
-
-def retry_with_exponential_backoff(max_retries: int = 3, initial_delay: int = 2, backoff_factor: int = 2):
-    """Decorator for retry logic with exponential backoff for API rate limits."""
-    def decorator(func):
-        @wraps(func)
-        def wrapper(*args, **kwargs):
-            last_exception = None
-            for attempt in range(max_retries + 1):
-                try:
-                    return func(*args, **kwargs)
-                except Exception as e:
-                    last_exception = e
-                    error_str = str(e)
-                    is_rate_limit = ("429" in error_str or "RESOURCE_EXHAUSTED" in error_str or "quota" in error_str.lower())
-                    if is_rate_limit and attempt < max_retries:
-                        delay = initial_delay * (backoff_factor ** attempt)
-                        logger.warning(f"Rate limit encountered. Retry {attempt + 1}/{max_retries} in {delay}s: {error_str}")
-                        time.sleep(delay)
-                    else:
-                        if is_rate_limit and attempt == max_retries:
-                            logger.error(f"Rate limit exhausted after {max_retries} retries")
-                        raise
-            raise last_exception
-        return wrapper
-    return decorator
-
-class Settings(BaseSettings):
-    google_api_key: str = os.getenv("GOOGLE_API_KEY")
-    hf_api_token: str = os.getenv("HF_API_TOKEN")
-    model_name: str = "gemini-2.5-flash-lite"
-    temperature: float = 0.0
-    chunk_size: int = 1000
-    chunk_overlap: int = 500
-    embedding_model: str = "sentence-transformers/all-MiniLM-L6-v2"
-
-settings = Settings()
-
-class BOQProcessor:
-    def __init__(self):
-        self.settings = settings
-        self.llm = GoogleGenerativeAI(model=self.settings.model_name, temperature=self.settings.temperature)
-        self.text_splitter = RecursiveCharacterTextSplitter(
-            chunk_size=self.settings.chunk_size,
-            chunk_overlap=self.settings.chunk_overlap
-        )
-
-    def _table_to_markdown(self, table: dict) -> str:
-        # Commented out: Table extraction not needed for now
-        """
-        headers = table.get('headers', [])
-        rows = table.get('rows', [])
-        if not headers:
-            return ''
-        md = '| ' + ' | '.join(headers) + ' |\n'
-        md += '|' + '|'.join(['---'] * len(headers)) + '|\n'
-        for row in rows:
-            md += '| ' + ' | '.join(str(cell) for cell in row) + ' |\n'
-        return md
-        """
-
-    def _call_extract_text_api(self, pdf_path: str, start_page: int = 1, end_page: int = 100, filename: str = None) -> str:
-        display_name = filename or os.path.basename(pdf_path)
-        logger.info(f'Starting text extraction for {display_name} (pages {start_page}-{end_page})')
-        with open(pdf_path, 'rb') as f:
-            files = {'file': (os.path.basename(pdf_path), f, 'application/pdf')}
-            data = {'start_page': start_page, 'end_page': end_page, 'filename': os.path.basename(pdf_path)}
-            headers = {'Authorization': f'Bearer {self.settings.hf_api_token}'}
-            response = requests.post(
-                'https://point9-extract-text-and-table.hf.space/api/text',
-                files=files,
-                data=data,
-                headers=headers
-            )
-            response.raise_for_status()
-            json_response = response.json()
-            if isinstance(json_response, dict):
-                result = json_response.get('result', '')
-            else:
-                logger.error(f"Unexpected response format: {json_response}")
-                result = ''
-            logger.info(f'Text extraction completed, response length: {len(result)}')
-            return result
-
-    def _call_extract_tables_api(self, pdf_path: str, start_page: int = 1, end_page: int = 2, filename: str = None) -> List[dict]:
-        # Commented out: Table extraction not needed for now
-        """
-        display_name = filename or os.path.basename(pdf_path)
-        logger.info(f'Starting table extraction for {display_name} (pages {start_page}-{end_page})')
-        with open(pdf_path, 'rb') as f:
-            files = {'file': (os.path.basename(pdf_path), f, 'application/pdf')}
-            data = {'start_page': start_page, 'end_page': end_page, 'filename': os.path.basename(pdf_path)}
-            headers = {'Authorization': f'Bearer {self.settings.hf_api_token}'}
-            response = requests.post(
-                'https://point9-extract-text-and-table.hf.space/api/tables',
-                files=files,
-                data=data,
-                headers=headers
-            )
-            response.raise_for_status()
-            json_response = response.json()
-            if isinstance(json_response, dict):
-                result = json_response.get('result', [])
-                # Filter to only include valid table dicts
-                valid_tables = [t for t in result if isinstance(t, dict)]
-                invalid_count = len(result) - len(valid_tables)
-                if invalid_count > 0:
-                    logger.warning(f"Filtered out {invalid_count} invalid tables (not dicts)")
-                result = valid_tables
-            else:
-                logger.error(f"Unexpected response format: {json_response}")
-                result = []
-            logger.info(f'Table extraction completed, found {len(result)} valid tables')
-            return result
-        """
-
-    def load_and_process_pdf(self, pdf_path: str, filename: str = None) -> List[Document]:
-        try:
-            display_name = filename or os.path.basename(pdf_path)
-            logger.info(f'Processing PDF from {display_name} using Hugging Face API')
-            logger.info('Calling text extraction API...')
-            extracted_text = self._call_extract_text_api(pdf_path, filename=filename)
-            logger.info(f'Extracted text length: {len(extracted_text)}')
-            if extracted_text:
-                logger.info(f'Text preview: {extracted_text[:200]}...')
-            else:
-                logger.warning('Extracted text is empty')
-            # Commented out: Table extraction not needed for now
-            """
-            logger.info('Calling table extraction API...')
-            tables = self._call_extract_tables_api(pdf_path, filename=filename)
-            logger.info(f'Extracted {len(tables)} tables')
-            logger.info('Converting tables to markdown...')
-            table_texts = [self._table_to_markdown(table) for table in tables]
-            logger.info(f'Converted {len(table_texts)} tables to markdown')
-            full_content = extracted_text + '\n\n' + '\n\n'.join(table_texts)
-            """
-            full_content = extracted_text
-            logger.info(f'Combined content length: {len(full_content)}')
-            logger.info('Splitting content into chunks...')
-            chunks = self.text_splitter.create_documents([full_content])
-            logger.info(f'Split into {len(chunks)} chunks')
-            return chunks
-        except Exception as e:
-            logger.error(f'Error loading and processing PDF: {e}')
-            raise
-
-    def create_vector_store(self, chunks: List[Document]) -> FAISS:
-        try:
-            logger.info('Creating embeddings and vector store')
-            logger.info(f'Processing {len(chunks)} chunks for embeddings')
-            embeddings = HuggingFaceEmbeddings(model_name=self.settings.embedding_model)
-            logger.info('Embeddings model loaded, creating FAISS vector store...')
-            vector_store = FAISS.from_documents(chunks, embeddings)
-            logger.info('Vector store created successfully')
-            return vector_store
-        except Exception as e:
-            logger.error(f'Error creating vector store: {e}')
-            raise
-
-    def setup_rag_chain(self, vector_store: FAISS):
-        logger.info('Setting up RAG chain with LangChain classic API (0.1.x)')
-        from langchain_classic.chains import ConversationalRetrievalChain
-        from langchain_classic.memory import ConversationBufferMemory
-        from langchain_core.prompts import PromptTemplate
-        memory = ConversationBufferMemory(memory_key='chat_history', return_messages=True)
-        qa_template = '''You are an expert assistant specializing in construction and tender documents, with deep knowledge of Bill of Quantities (BOQ) analysis. Your role is to provide accurate, helpful, and professional responses based solely on the provided context.
-
-Guidelines:
-- Always base your answers on the given context. Do not use external knowledge or assumptions.
-- For BOQ-related questions, provide detailed, structured information including item codes, descriptions, quantities, units, rates, and amounts where available.
-- If the context lacks specific information, respond with: "The requested information is not available in the provided document context."
-- Be concise yet comprehensive. Structure responses clearly (e.g., use bullet points or tables for lists).
-- Handle follow-up questions by referencing previous context in the conversation history.
-- Maintain neutrality and professionalism in all responses.
-
-{context}
-
-Question: {question}
-Answer:'''
-        qa_prompt = PromptTemplate.from_template(qa_template)
-        qa_chain = ConversationalRetrievalChain.from_llm(
-            llm=self.llm,
-            retriever=vector_store.as_retriever(),
-            memory=memory,
-            combine_docs_chain_kwargs={'prompt': qa_prompt},
-        )
-        logger.info('RAG chain set up successfully with LangChain classic API')
-        return qa_chain
-
-    def _extract_metadata(self, chunks: List[Document]) -> str:
-        metadata_text = '\n\n'.join([chunk.page_content for chunk in chunks[:3]])
-        metadata_prompt = f'''Extract key information from this tender document excerpt in a concise format:
-- Document Type
-- Project Name
-- Issuing Authority
-- Tender Number
-- Date
-- Location
-
-Document excerpt:
-{metadata_text[:2000]}
-
-Output only the facts, no extra analysis.'''
-        logger.info('Invoking LLM for metadata extraction...')
-        result = str(self.llm.invoke(metadata_prompt))
-        logger.info('Metadata extraction completed')
-        return result
-
-    def _batch_chunks(self, chunks: List[Document], batch_size: int = 24) -> List[List[Document]]:
-        return [chunks[i:i + batch_size] for i in range(0, len(chunks), batch_size)]
-
-    def _extract_boq_from_batch(self, batch_text: str, batch_chunks: List[Document], batch_num: int) -> List[str]:
-        extraction_prompt = '''Analyze this text and extract ONLY Bill of Quantities (BOQ) line items if present.
-
-Look for structured data with:
-- Item numbers or codes
-- Descriptions of work/materials (extract the complete, full description as it appears in the document, without truncation)
-- Quantities
-- Units (Nos, Sqm, Cum, m, etc.)
-- Rates/Unit prices
-- Total amounts
-
-If you find BOQ items, return them in this EXACT format (pipe-separated):
-ITEM_CODE|DESCRIPTION|QUANTITY|UNIT|RATE|AMOUNT|CONFIDENCE
-
-Where:
-- CONFIDENCE is a score (0-100%) based on how clearly and completely the data appears in the text. Use lower scores (e.g., 70-90%) if information is partially missing, inferred, or unclear. Use 100% only for complete, directly stated data.
-
-Rules for columns:
-- If an entire column has no values, omit that column.
-- For missing values, use "NA".
-
-Return multiple items on separate lines. If NO BOQ items are found, return: "NO_BOQ_ITEMS"
-
-Text to analyze:
-{batch_text}
-
-Extract only actual BOQ line items.'''
-
-        prompt_text = batch_text[:30000]
-        prompt = extraction_prompt.format(batch_text=prompt_text)
-
-        try:
-            logger.info(f'Invoking LLM for BOQ extraction on batch {batch_num}...')
-            result = self.llm.invoke(prompt)
-            logger.info(f'LLM response received for batch {batch_num}')
-            if 'NO_BOQ_ITEMS' in str(result):
-                logger.info(f'No BOQ items found in batch {batch_num}')
-                return []
-            boq_items = []
-            lines = str(result).strip().split('\n')
-            for line in lines:
-                line = line.strip()
-                if not line or '|' not in line:
-                    continue
-                parts = [p.strip() for p in line.split('|')]
-                if len(parts) < 7:
-                    parts += ['NA'] * (7 - len(parts))
-                # Temp source
-                parts.append("Unknown")
-                # Per-item page detection
-                desc = parts[1]
-                search_str = desc[:30].strip().lower()
-                batch_text_lower = batch_text.lower()
-                pos = batch_text_lower.rfind(search_str)
-                if pos != -1:
-                    marker_pattern = r"(?i)(?:---\s*)?page\s+(\d+)(?:\s*---)?"
-                    matches = list(re.finditer(marker_pattern, batch_text[:pos]))
-                    if matches:
-                        page = matches[-1].group(1)
-                        parts[7] = f"Page {page}"
-                boq_items.append('|'.join(parts[:8]))
-            logger.info(f'Extracted {len(boq_items)} BOQ items from batch {batch_num}')
-            return boq_items
-        except Exception as e:
-            logger.warning(f'Error processing batch {batch_num}: {e}')
-            return []
-
-    def _format_boq_output(self, unique_items: List[str], metadata_result: str) -> str:
-        logger.info('Formatting BOQ output...')
-        if not unique_items:
-            logger.info('No BOQ items to format')
-            return f'''## DOCUMENT SUMMARY
-{metadata_result}
-
-## DETAILED BILL OF QUANTITIES
-No BOQ items were found in this document.'''
-
-        col_headers = ['Item No/Code', 'Description', 'Quantity', 'Unit', 'Rate', 'Amount', 'Confidence Score', 'Source']
-        cols_present = [False] * 8
-        normalized_items = []
-        for item in unique_items:
-            parts = [p.strip() for p in item.split('|')]
-            if len(parts) < 8:
-                parts += ['NA'] * (8 - len(parts))
-            normalized_items.append(parts[:8])
-            for i in range(8):
-                if parts[i] and parts[i].upper() != 'NA':
-                    cols_present[i] = True
-
-        col_indices = [i for i, present in enumerate(cols_present) if present]
-        if 0 not in col_indices:
-            col_indices.insert(0, 0)
-        if 1 not in col_indices:
-            col_indices.insert(1, 1)
-
-        header_row = '| ' + ' | '.join([col_headers[i] for i in col_indices]) + ' |\n'
-        sep_row = '|' + '|'.join(['-' * (len(col_headers[i]) + 2) for i in col_indices]) + '|\n'
-
-        formatted_boq = f'''## DOCUMENT SUMMARY
-{metadata_result}
-
-## DETAILED BILL OF QUANTITIES
-**Total Items Found:** {len(unique_items)}
-
-{header_row}{sep_row}'''
-
-        for parts in normalized_items:
-            # parts[1] remains full for complete descriptions
-            parts[7] = parts[7][:50] if len(parts[7]) > 50 else parts[7]  # Truncate source if too long
-            row_vals = [parts[i] for i in col_indices]
-            # Add % to confidence score if present
-            if 6 in col_indices:
-                conf_idx = col_indices.index(6)
-                if row_vals[conf_idx] != 'NA':
-                    row_vals[conf_idx] = row_vals[conf_idx].rstrip('%') + '%'
-                    # Align confidence with source: if source unknown, set confidence to N/A
-                    if parts[7] == "Unknown":
-                        row_vals[conf_idx] = "N/A"
-            formatted_boq += '| ' + ' | '.join(row_vals) + ' |\n'
-
-        # formatted_boq += f'\n## SUMMARY\n- **Total Items:** {len(unique_items)}\n'
-
-        try:
-            s = formatted_boq.replace('\r\n', '\n').replace('\r', '\n')
-            lines = [ln.lstrip() for ln in s.split('\n')]
-            header_idx = next((i for i, ln in enumerate(lines) if ln.startswith('| ')), None)
-            if header_idx and header_idx > 0 and lines[header_idx - 1].strip():
-                lines.insert(header_idx, '')
-            if header_idx:
-                sep_idx = header_idx + 1
-                if not (sep_idx < len(lines) and re.match(r'^\|\s*-+', lines[sep_idx])):
-                    cols = [c for c in lines[header_idx].split('|') if c.strip()]
-                    sep = '|' + '|'.join(['---' for _ in cols]) + '|'
-                    lines.insert(sep_idx, sep)
-            formatted_boq = '\n'.join(lines).strip() + '\n\n'
-        except Exception:
-            pass
-
-        return formatted_boq
-
-    @retry_with_exponential_backoff(max_retries=3, initial_delay=2)
-    def extract_boq_comprehensive(self, chunks: List[Document], vector_store: FAISS = None) -> str:
-        try:
-            logger.info(f'Starting comprehensive BOQ extraction from {len(chunks)} chunks')
-            logger.info('Extracting document metadata...')
-            metadata_result = self._extract_metadata(chunks)
-            logger.info('Metadata extracted, creating batches...')
-            batches = self._batch_chunks(chunks)
-            logger.info(f'Created {len(batches)} batches')
-            boq_items = []
-            for batch_num, batch_chunks in enumerate(batches, 1):
-                logger.info(f'Processing batch {batch_num}/{len(batches)} ({len(batch_chunks)} chunks)')
-                chunk_texts = [chunk.page_content for chunk in batch_chunks]
-                batch_text = '\n\n'.join(chunk_texts)
-                logger.info(f'Batch text length: {len(batch_text)}')
-                batch_items = self._extract_boq_from_batch(batch_text, batch_chunks, batch_num)
-                boq_items.extend(batch_items)
-                logger.info(f'Batch {batch_num} yielded {len(batch_items)} items')
-            unique_items = list(dict.fromkeys(boq_items))
-            logger.info(f'Found {len(unique_items)} unique BOQ items after deduplication')
-            logger.info('Formatting BOQ output...')
-            formatted_boq = self._format_boq_output(unique_items, metadata_result)
-            logger.info('Comprehensive BOQ extraction completed successfully')
-            return formatted_boq
-        except Exception as e:
-            logger.error(f'Error in comprehensive BOQ extraction: {e}')
-            raise
-
-def check_consistency(chunks: List[Document], vector_store: FAISS, runs: int = 4) -> dict:
-    """Run extraction multiple times and compute variance."""
-    from difflib import SequenceMatcher
-    
-    results = []
-    for _ in range(runs):
-        try:
-            boq = extract_boq_comprehensive(chunks, vector_store)
-            results.append(boq)
-        except Exception as e:
-            logger.warning(f"Consistency run failed: {e}")
-            results.append("")
-    
-    # Variance: Average similarity between pairs
-    similarities = []
-    for i in range(len(results)):
-        for j in range(i+1, len(results)):
-            if results[i] and results[j]:
-                sim = SequenceMatcher(None, results[i], results[j]).ratio()
-                similarities.append(sim)
-    
-    avg_similarity = sum(similarities) / len(similarities) if similarities else 0
-    consistency_score = avg_similarity * 100
-    
-    # Average confidence from per-item scores
-    all_confidences = []
-    for boq in results:
-        if boq:
-            lines = boq.split('\n')
-            confidence_idx = None
-            for line in lines:
-                line = line.strip()
-                if '|' in line and 'Confidence' in line and not line.startswith('| ---'):
-                    # Header row: find index of Confidence
-                    parts = [p.strip() for p in line.split('|')[1:-1]]
-                    confidence_idx = next((i for i, p in enumerate(parts) if 'Confidence' in p), None)
-                    if confidence_idx is not None:
-                        break
-            if confidence_idx is not None:
-                for line in lines:
-                    if '|' in line and not line.startswith('| ---') and 'Confidence' not in line:
-                        parts = [p.strip() for p in line.split('|')[1:-1]]
-                        if len(parts) > confidence_idx:
-                            try:
-                                conf_str = parts[confidence_idx]
-                                if conf_str and conf_str != 'NA':
-                                    # Remove % if present
-                                    conf_str = conf_str.rstrip('%')
-                                    conf = float(conf_str)
-                                    all_confidences.append(conf)
-                            except (ValueError, IndexError):
-                                pass
-    
-    avg_confidence = sum(all_confidences) / len(all_confidences) if all_confidences else 0
-    
-    return {
-        "consistency_score": round(consistency_score, 2),
-        "runs": runs,
-        "successful_runs": len([r for r in results if r]),
-        "avg_similarity": round(avg_similarity, 2),
-        "avg_confidence": round(avg_confidence, 2),  # New metric
-        "total_confidence_scores": len(all_confidences)
-    }
-
-# Global instance for backward compatibility
-processor = BOQProcessor()
-
-# Backward compatibility functions
-def load_and_process_pdf(pdf_path: str, filename: str = None) -> List[Document]:
-    return processor.load_and_process_pdf(pdf_path, filename)
-
-def create_vector_store(chunks: List[Document]) -> FAISS:
-    return processor.create_vector_store(chunks)
-
-def setup_rag_chain(vector_store: FAISS):
-    return processor.setup_rag_chain(vector_store)
-
-@retry_with_exponential_backoff(max_retries=3, initial_delay=2)
-def extract_boq_comprehensive(chunks: List[Document], vector_store: FAISS = None) -> str:
-    return processor.extract_boq_comprehensive(chunks, vector_store)

config/settings.py

@@ -0,0 +1,109 @@
+"""
+Centralized configuration management for BOQTenders.
+All configurable parameters are defined here with sensible defaults.
+"""
+import os
+from typing import Optional
+from pydantic_settings import BaseSettings
+from pydantic import Field
+import dotenv
+
+# Load environment variables from .env file
+dotenv.load_dotenv()
+
+
+class LLMSettings(BaseSettings):
+    """LLM-related configuration."""
+    
+    # API Keys
+    google_api_key: str = Field(default_factory=lambda: os.getenv("GOOGLE_API_KEY"),description="Google API key for Gemini")
+    
+    # Model Configuration
+    model_name: str = Field(default="gemini-2.5-flash-lite",description="LLM model name to use")
+    temperature: float = Field(default=0.0,ge=0.0,le=2.0,description="LLM temperature (0.0 = deterministic)")
+    max_output_tokens: int = Field(default=8192,description="Maximum tokens in LLM response")
+
+
+class EmbeddingSettings(BaseSettings):
+    """Embedding and vector store configuration."""
+    
+    embedding_model: str = Field(default="sentence-transformers/all-MiniLM-L6-v2",description="HuggingFace embedding model name")
+    
+    # Text Splitting
+    chunk_size: int = Field(default=1000,ge=100,le=10000,description="Size of text chunks for splitting")
+    chunk_overlap: int = Field(default=500,ge=0,description="Overlap between consecutive chunks")
+
+
+class PDFExtractionSettings(BaseSettings):
+    """PDF extraction API configuration."""
+    
+    hf_api_token: str = Field(default_factory=lambda: os.getenv("HF_API_TOKEN"),description="HuggingFace API token")
+    extraction_api_url: str = Field(default="https://point9-extract-text-and-table.hf.space/api/text",description="URL for PDF text extraction API")
+    
+    start_page: int = Field(default=1,ge=1,description="Default start page for extraction")
+    end_page: int = Field(default=100,ge=1,description="Default end page for extraction")
+    request_timeout: int = Field(default=120,description="API request timeout in seconds")
+
+
+class BOQExtractionSettings(BaseSettings):
+    """BOQ extraction specific configuration."""
+    
+    batch_size: int = Field(default=25,ge=1,le=100,description="Number of chunks per batch for BOQ extraction")
+    max_prompt_length: int = Field(default=30000,description="Maximum characters in extraction prompt")
+    page_search_length: int = Field(default=30,description="Characters to use for page detection search")
+    source_max_length: int = Field(default=50,description="Maximum length for source column")
+
+
+class ConsistencySettings(BaseSettings):
+    """Consistency check configuration."""
+    
+    default_runs: int = Field(default=4,ge=2,le=10,description="Default number of runs for consistency check")
+    low_consistency_threshold: float = Field(default=80.0,ge=0.0,le=100.0,description="Threshold below which consistency is considered low")
+
+
+class APISettings(BaseSettings):
+    """FastAPI server configuration."""
+    
+    title: str = Field(default="BOQ Chatbot API",description="API title")
+    description: str = Field(default="API for extracting and querying BOQ from tender PDFs using RAG",description="API description")
+    version: str = Field(default="1.0.0",description="API version")
+    host: str = Field(default="0.0.0.0",description="Server host")
+    port: int = Field(default=8000,description="Server port")
+    debug: bool = Field(default=False,description="Enable debug mode")
+    docs_enabled: bool = Field(default=True,description="Enable API documentation endpoints")
+    cors_origins: list = Field(default=["*"],description="Allowed CORS origins")
+
+
+class StreamlitSettings(BaseSettings):
+    """Streamlit UI configuration."""
+    
+    page_title: str = Field(default="BOQ Agent",description="Page title")
+    page_icon: str = Field(default="📄",description="Page icon")
+    layout: str = Field(default="wide",description="Page layout (wide/centered)")
+
+
+class Settings(BaseSettings):
+    """
+    Main settings class that aggregates all configuration sections.
+    Access via: settings.llm, settings.embedding, settings.pdf, etc.
+    """
+    
+    # Global settings
+    log_level: str = Field(default="INFO",description="Logging level (DEBUG, INFO, WARNING, ERROR)")
+    
+    llm: LLMSettings = Field(default_factory=LLMSettings)
+    embedding: EmbeddingSettings = Field(default_factory=EmbeddingSettings)
+    pdf: PDFExtractionSettings = Field(default_factory=PDFExtractionSettings)
+    boq: BOQExtractionSettings = Field(default_factory=BOQExtractionSettings)
+    consistency: ConsistencySettings = Field(default_factory=ConsistencySettings)
+    api: APISettings = Field(default_factory=APISettings)
+    streamlit: StreamlitSettings = Field(default_factory=StreamlitSettings)
+    
+    class Config:
+        env_file = ".env"
+        env_file_encoding = "utf-8"
+        extra = "ignore"  # Ignore extra env vars not defined in model
+
+
+# Global settings instance
+settings = Settings()

core/embeddings.py

@@ -0,0 +1,118 @@
+"""
+Embedding and vector store management module.
+"""
+from typing import List, Optional
+from loguru import logger
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_huggingface import HuggingFaceEmbeddings
+from langchain_community.vectorstores import FAISS
+from langchain_core.documents import Document
+
+from config.settings import settings
+
+
+class EmbeddingService:
+    """
+    Handles text chunking, embeddings, and vector store operations.
+    
+    Example:
+        service = EmbeddingService()
+        chunks = service.split_text(text)
+        vector_store = service.create_vector_store(chunks)
+    """
+    
+    def __init__(self, embedding_model: str = None, chunk_size: int = None, chunk_overlap: int = None):
+        """
+        Initialize embedding service.
+        
+        Args:
+            embedding_model: HuggingFace model name. Defaults to config value.
+            chunk_size: Size of text chunks. Defaults to config value.
+            chunk_overlap: Overlap between chunks. Defaults to config value.
+        """
+        self.embedding_model = embedding_model or settings.embedding.embedding_model
+        self.chunk_size = chunk_size or settings.embedding.chunk_size
+        self.chunk_overlap = chunk_overlap or settings.embedding.chunk_overlap
+        
+        self._text_splitter = RecursiveCharacterTextSplitter(
+            chunk_size=self.chunk_size,
+            chunk_overlap=self.chunk_overlap
+        )
+        self._embeddings: Optional[HuggingFaceEmbeddings] = None
+    
+    @property
+    def embeddings(self) -> HuggingFaceEmbeddings:
+        """Lazy-load embeddings model."""
+        if self._embeddings is None:
+            logger.info(f'Loading embeddings model: {self.embedding_model}')
+            self._embeddings = HuggingFaceEmbeddings(model_name=self.embedding_model)
+            logger.info('Embeddings model loaded successfully')
+        return self._embeddings
+    
+    def split_text(self, text: str) -> List[Document]:
+        """
+        Split text into chunks for processing.
+        
+        Args:
+            text: Full text content to split.
+        
+        Returns:
+            List of Document chunks.
+        """
+        logger.info(f'Splitting text of length {len(text)} into chunks...')
+        chunks = self._text_splitter.create_documents([text])
+        logger.info(f'Split into {len(chunks)} chunks')
+        return chunks
+    
+    def create_vector_store(self, chunks: List[Document]) -> FAISS:
+        """
+        Create a FAISS vector store from document chunks.
+        
+        Args:
+            chunks: List of Document objects.
+        
+        Returns:
+            FAISS vector store instance.
+        
+        Raises:
+            Exception: If vector store creation fails.
+        """
+        try:
+            logger.info(f'Creating vector store from {len(chunks)} chunks')
+            vector_store = FAISS.from_documents(chunks, self.embeddings)
+            logger.info('Vector store created successfully')
+            return vector_store
+        except Exception as e:
+            logger.error(f'Error creating vector store: {e}')
+            raise
+    
+    def add_documents(self, vector_store: FAISS, documents: List[Document]) -> None:
+        """
+        Add new documents to an existing vector store.
+        
+        Args:
+            vector_store: Existing FAISS vector store.
+            documents: Documents to add.
+        """
+        logger.info(f'Adding {len(documents)} documents to vector store')
+        vector_store.add_documents(documents)
+        logger.info('Documents added successfully')
+    
+    def similarity_search(
+        self,
+        vector_store: FAISS,
+        query: str,
+        k: int = 4
+    ) -> List[Document]:
+        """
+        Perform similarity search on vector store.
+        
+        Args:
+            vector_store: FAISS vector store to search.
+            query: Search query.
+            k: Number of results to return.
+        
+        Returns:
+            List of most similar documents.
+        """
+        return vector_store.similarity_search(query, k=k)

core/llm.py

@@ -0,0 +1,78 @@
+"""
+LLM client wrapper for Google Generative AI (Gemini).
+"""
+from typing import Optional
+from loguru import logger
+from langchain_google_genai import GoogleGenerativeAI
+
+from config.settings import settings
+
+
+class LLMClient:
+    """
+    Wrapper for LLM interactions with Google Generative AI.
+    
+    Example:
+        client = LLMClient()
+        response = client.invoke("What is a BOQ?")
+    """
+    
+    def __init__(self, model_name: str = None, temperature: float = None, api_key: str = None):
+        """
+        Initialize LLM client.
+        
+        Args:
+            model_name: Model to use. Defaults to config value.
+            temperature: Sampling temperature. Defaults to config value.
+            api_key: Google API key. Defaults to config value.
+        """
+        self.model_name = model_name or settings.llm.model_name
+        self.temperature = temperature if temperature is not None else settings.llm.temperature
+        self.api_key = api_key or settings.llm.google_api_key
+        
+        self._llm: Optional[GoogleGenerativeAI] = None
+    
+    @property
+    def llm(self) -> GoogleGenerativeAI:
+        """Lazy-load LLM instance."""
+        if self._llm is None:
+            logger.info(f'Initializing LLM: {self.model_name} (temp={self.temperature})')
+            self._llm = GoogleGenerativeAI(
+                model=self.model_name,
+                temperature=self.temperature,
+                google_api_key=self.api_key
+            )
+            logger.info('LLM initialized successfully')
+        return self._llm
+    
+    def invoke(self, prompt: str) -> str:
+        """
+        Send a prompt to the LLM and get a response.
+        
+        Args:
+            prompt: The prompt text.
+        
+        Returns:
+            LLM response as string.
+        """
+        logger.debug(f'Invoking LLM with prompt of length {len(prompt)}')
+        result = str(self.llm.invoke(prompt))
+        logger.debug(f'LLM response received, length: {len(result)}')
+        return result
+    
+    def batch_invoke(self, prompts: list[str]) -> list[str]:
+        """
+        Send multiple prompts to the LLM.
+        
+        Args:
+            prompts: List of prompt texts.
+        
+        Returns:
+            List of LLM responses.
+        """
+        logger.info(f'Batch invoking LLM with {len(prompts)} prompts')
+        results = []
+        for i, prompt in enumerate(prompts, 1):
+            logger.debug(f'Processing prompt {i}/{len(prompts)}')
+            results.append(self.invoke(prompt))
+        return results

core/pdf_extractor.py

@@ -0,0 +1,101 @@
+"""
+PDF text extraction module using external API.
+"""
+import os
+from typing import Optional
+import requests
+from loguru import logger
+
+from config.settings import settings
+
+
+class PDFExtractor:
+    """
+    Handles PDF text extraction using HuggingFace Space API.
+    
+    Example:
+        extractor = PDFExtractor()
+        text = extractor.extract_text("document.pdf")
+    """
+    
+    def __init__(self, api_url: str = None, api_token: str = None, timeout: int = None):
+        """
+        Initialize PDF extractor.
+        
+        Args:
+            api_url: URL for extraction API. Defaults to config value.
+            api_token: HuggingFace API token. Defaults to config value.
+            timeout: Request timeout in seconds. Defaults to config value.
+        """
+        self.api_url = api_url or settings.pdf.extraction_api_url
+        self.api_token = api_token or settings.pdf.hf_api_token
+        self.timeout = timeout or settings.pdf.request_timeout
+    
+    def extract_text(self, pdf_path: str, start_page: int = None, end_page: int = None, filename: str = None) -> str:
+        """
+        Extract text from a PDF file.
+        
+        Args:
+            pdf_path: Path to the PDF file.
+            start_page: Starting page number (1-indexed). Defaults to config value.
+            end_page: Ending page number. Defaults to config value.
+            filename: Display name for logging. Defaults to basename of pdf_path.
+        
+        Returns:
+            Extracted text content.
+        
+        Raises:
+            requests.RequestException: If API request fails.
+            FileNotFoundError: If PDF file doesn't exist.
+        """
+        # Use config defaults if not specified
+        start_page = start_page or settings.pdf.start_page
+        end_page = end_page or settings.pdf.end_page
+        display_name = filename or os.path.basename(pdf_path)
+        
+        logger.info(f'Starting text extraction for {display_name} (pages {start_page}-{end_page})')
+        
+        if not os.path.exists(pdf_path):
+            raise FileNotFoundError(f"PDF file not found: {pdf_path}")
+        
+        with open(pdf_path, 'rb') as f:
+            files = {'file': (os.path.basename(pdf_path), f, 'application/pdf')}
+            data = {
+                'start_page': start_page,
+                'end_page': end_page,
+                'filename': os.path.basename(pdf_path)
+            }
+            headers = {'Authorization': f'Bearer {self.api_token}'}
+            
+            response = requests.post(
+                self.api_url,
+                files=files,
+                data=data,
+                headers=headers,
+                timeout=self.timeout
+            )
+            response.raise_for_status()
+            
+            json_response = response.json()
+            if isinstance(json_response, dict):
+                result = json_response.get('result', '')
+            else:
+                logger.error(f"Unexpected response format: {json_response}")
+                result = ''
+            
+            logger.info(f'Text extraction completed, response length: {len(result)}')
+            return result
+    
+    def extract_text_preview(self, pdf_path: str, max_chars: int = 200) -> str:
+        """
+        Extract and return a preview of the PDF text.
+        
+        Args:
+            pdf_path: Path to the PDF file.
+            max_chars: Maximum characters to return.
+        
+        Returns:
+            Preview of extracted text.
+        """
+        text = self.extract_text(pdf_path, start_page=1, end_page=5)
+        return text[:max_chars] + "..." if len(text) > max_chars else text

core/rag_chain.py

@@ -0,0 +1,80 @@
+"""
+RAG chain builder for conversational retrieval.
+"""
+from loguru import logger
+from langchain_community.vectorstores import FAISS
+from langchain_classic.chains import ConversationalRetrievalChain
+from langchain_classic.memory import ConversationBufferMemory
+from langchain_core.prompts import PromptTemplate
+
+from core.llm import LLMClient
+from prompts.get_prompts import QA_TEMPLATE
+
+
+class RAGChainBuilder:
+    """
+    Builder for RAG (Retrieval-Augmented Generation) chains.
+    
+    Example:
+        builder = RAGChainBuilder()
+        chain = builder.build(vector_store)
+        response = chain({"question": "What is the total quantity?"})
+    """
+    
+    def __init__(self, llm_client: LLMClient = None):
+        """
+        Initialize RAG chain builder.
+        
+        Args:
+            llm_client: LLM client instance. Creates new one if not provided.
+        """
+        self.llm_client = llm_client or LLMClient()
+    
+    def build(self, vector_store: FAISS, qa_template: str = None, memory_key: str = "chat_history", return_messages: bool = True) -> ConversationalRetrievalChain:
+        """
+        Build a conversational retrieval chain.
+        
+        Args:
+            vector_store: FAISS vector store with document embeddings.
+            qa_template: Custom Q&A prompt template. Defaults to standard template.
+            memory_key: Key for conversation memory.
+            return_messages: Whether to return messages in memory.
+        
+        Returns:
+            Configured ConversationalRetrievalChain.
+        """
+        logger.info('Building RAG chain with LangChain classic API')
+        
+        # Create memory
+        memory = ConversationBufferMemory(
+            memory_key=memory_key,
+            return_messages=return_messages
+        )
+        
+        # Create prompt
+        template = qa_template or QA_TEMPLATE
+        qa_prompt = PromptTemplate.from_template(template)
+        
+        # Build chain
+        chain = ConversationalRetrievalChain.from_llm(
+            llm=self.llm_client.llm,
+            retriever=vector_store.as_retriever(),
+            memory=memory,
+            combine_docs_chain_kwargs={'prompt': qa_prompt},
+        )
+        
+        logger.info('RAG chain built successfully')
+        return chain
+    
+    def build_simple_retriever(self, vector_store: FAISS, k: int = 4):
+        """
+        Build a simple retriever without conversation memory.
+        
+        Args:
+            vector_store: FAISS vector store.
+            k: Number of documents to retrieve.
+        
+        Returns:
+            Retriever instance.
+        """
+        return vector_store.as_retriever(search_kwargs={"k": k})

prompts/get_prompts.py

@@ -0,0 +1,17 @@
+"""
+Load prompt templates from YAML file for BOQ extraction.
+"""
+from pathlib import Path
+import yaml
+
+# Load templates from YAML
+_TEMPLATES_PATH = Path(__file__).parent / "templates.yaml"
+
+with open(_TEMPLATES_PATH, 'r', encoding='utf-8') as f:
+    _templates = yaml.safe_load(f)
+
+# Export as module constants
+QA_TEMPLATE = _templates['qa_template']
+METADATA_EXTRACTION_TEMPLATE = _templates['metadata_extraction_template']
+BOQ_EXTRACTION_TEMPLATE = _templates['boq_extraction_template']
+BOQ_COLUMN_HEADERS = _templates['boq_column_headers']

prompts/templates.yaml

@@ -0,0 +1,71 @@
+# BOQTenders Prompt Templates
+# YAML format for better production management and versioning
+
+qa_template: |
+  You are an expert assistant specializing in construction and tender documents, with deep knowledge of Bill of Quantities (BOQ) analysis. Your role is to provide accurate, helpful, and professional responses based solely on the provided context.
+
+  Guidelines:
+  - Always base your answers on the given context. Do not use external knowledge or assumptions.
+  - For BOQ-related questions, provide detailed, structured information including item codes, descriptions, quantities, units, rates, and amounts where available.
+  - If the context lacks specific information, respond with: "The requested information is not available in the provided document context."
+  - Be concise yet comprehensive. Structure responses clearly (e.g., use bullet points or tables for lists).
+  - Handle follow-up questions by referencing previous context in the conversation history.
+  - Maintain neutrality and professionalism in all responses.
+
+  {context}
+
+  Question: {question}
+  Answer:
+
+metadata_extraction_template: |
+  Extract key information from this tender document excerpt in a concise format:
+  - Document Type
+  - Project Name
+  - Issuing Authority
+  - Tender Number
+  - Date
+  - Location
+
+  Document excerpt:
+  {document_text}
+
+  Output only the facts, no extra analysis.
+
+boq_extraction_template: |
+  Analyze this text and extract ONLY Bill of Quantities (BOQ) line items if present.
+
+  Look for structured data with:
+  - Item numbers or codes
+  - Descriptions of work/materials (extract the complete, full description as it appears in the document, without truncation)
+  - Quantities
+  - Units (Nos, Sqm, Cum, m, etc.)
+  - Rates/Unit prices
+  - Total amounts
+
+  If you find BOQ items, return them in this EXACT format (pipe-separated):
+  ITEM_CODE|DESCRIPTION|QUANTITY|UNIT|RATE|AMOUNT|CONFIDENCE
+
+  Where:
+  - CONFIDENCE is a score (0-100%) based on how clearly and completely the data appears in the text. Use lower scores (e.g., 70-90%) if information is partially missing, inferred, or unclear. Use 100% only for complete, directly stated data.
+
+  Rules for columns:
+  - If an entire column has no values, omit that column.
+  - For missing values, use "NA".
+
+  Return multiple items on separate lines. If NO BOQ items are found, return: "NO_BOQ_ITEMS"
+
+  Text to analyze:
+  {batch_text}
+
+  Extract only actual BOQ line items.
+
+# Column headers for BOQ output table
+boq_column_headers:
+  - "Item No/Code"
+  - "Description"
+  - "Quantity"
+  - "Unit"
+  - "Rate"
+  - "Amount"
+  - "Confidence Score"
+  - "Source"

requirements.txt

@@ -1,15 +1,51 @@
-langchain
-langchain-community
-pdfplumber
-faiss-cpu
-sentence-transformers
-streamlit
-langchain-google-genai
-langchain-huggingface
-fastapi
-uvicorn
-pydantic-settings
-loguru
-python-dotenv
-python-multipart
-requests
+# BOQTenders - Requirements
+# Python 3.9+
+
+# ============================================
+# Core Dependencies
+# ============================================
+python-dotenv>=1.0.0
+pydantic>=2.0.0
+pydantic-settings>=2.0.0
+loguru>=0.7.0
+PyYAML>=6.0.0
+
+# ============================================
+# LLM & AI
+# ============================================
+# LangChain 0.1.x (Classic API - stable)
+langchain>=0.1.0,<0.2.0
+langchain-community>=0.0.1,<0.1.0
+langchain-google-genai>=0.0.5
+
+# Vector Store
+faiss-cpu>=1.7.4
+
+# Embeddings
+sentence-transformers>=2.2.0
+
+# ============================================
+# PDF Processing
+# ============================================
+PyPDF2>=3.0.0
+requests>=2.31.0
+
+# ============================================
+# Web Framework
+# ============================================
+# API
+fastapi>=0.100.0
+uvicorn[standard]>=0.23.0
+python-multipart>=0.0.6
+
+# UI
+streamlit>=1.28.0
+
+# ============================================
+# Development (optional)
+# ============================================
+# pytest>=7.4.0
+# pytest-asyncio>=0.21.0
+# black>=23.0.0
+# isort>=5.12.0
+# mypy>=1.5.0

services/boq_extractor.py

@@ -0,0 +1,296 @@
+"""
+BOQ extraction service for extracting Bill of Quantities from documents.
+"""
+import re
+from typing import List, Optional, Tuple
+from loguru import logger
+from langchain_core.documents import Document
+from langchain_community.vectorstores import FAISS
+
+from config.settings import settings
+from core.llm import LLMClient
+from prompts.get_prompts import (
+    METADATA_EXTRACTION_TEMPLATE,
+    BOQ_EXTRACTION_TEMPLATE,
+    BOQ_COLUMN_HEADERS,
+)
+
+# Page marker pattern for detecting source pages
+PAGE_MARKER_PATTERN = r"(?i)(?:---\s*)?page\s+(\d+)(?:\s*---)?"
+
+
+class BOQExtractor:
+    """
+    Extracts Bill of Quantities (BOQ) data from document chunks.
+    
+    Example:
+        extractor = BOQExtractor()
+        boq_output = extractor.extract(chunks)
+    """
+    
+    def __init__(self, llm_client: LLMClient = None, batch_size: int = None, max_prompt_length: int = None, page_search_length: int = None):
+        """
+        Initialize BOQ extractor.
+        
+        Args:
+            llm_client: LLM client instance. Creates new one if not provided.
+            batch_size: Number of chunks per batch. Defaults to config value.
+            max_prompt_length: Max chars in prompt. Defaults to config value.
+            page_search_length: Chars for page detection. Defaults to config value.
+        """
+        self.llm_client = llm_client or LLMClient()
+        self.batch_size = batch_size or settings.boq.batch_size
+        self.max_prompt_length = max_prompt_length or settings.boq.max_prompt_length
+        self.page_search_length = page_search_length or settings.boq.page_search_length
+        self.source_max_length = settings.boq.source_max_length
+    
+    def _batch_chunks(self, chunks: List[Document]) -> List[List[Document]]:
+        """Split chunks into batches."""
+        return [
+            chunks[i:i + self.batch_size]
+            for i in range(0, len(chunks), self.batch_size)
+        ]
+    
+    def _extract_metadata(self, chunks: List[Document]) -> str:
+        """Extract document metadata from first few chunks."""
+        metadata_text = '\n\n'.join([chunk.page_content for chunk in chunks[:3]])
+        prompt = METADATA_EXTRACTION_TEMPLATE.format(
+            document_text=metadata_text[:2000]
+        )
+        
+        logger.info('Invoking LLM for metadata extraction...')
+        result = self.llm_client.invoke(prompt)
+        logger.info('Metadata extraction completed')
+        return result
+    
+    def _detect_page_source(self, desc: str, batch_text: str) -> str:
+        """
+        Detect the page number for a BOQ item based on its description.
+        
+        Args:
+            desc: Item description.
+            batch_text: Full batch text to search in.
+        
+        Returns:
+            Page source string (e.g., "Page 5") or "Unknown".
+        """
+        search_str = desc[:self.page_search_length].strip().lower()
+        batch_text_lower = batch_text.lower()
+        pos = batch_text_lower.rfind(search_str)
+        
+        if pos != -1:
+            matches = list(re.finditer(PAGE_MARKER_PATTERN, batch_text[:pos]))
+            if matches:
+                page = matches[-1].group(1)
+                return f"Page {page}"
+        
+        return "Unknown"
+    
+    def _parse_boq_line(self, line: str, batch_text: str) -> Optional[str]:
+        """
+        Parse a single BOQ line from LLM output.
+        
+        Args:
+            line: Raw line from LLM output.
+            batch_text: Full batch text for page detection.
+        
+        Returns:
+            Formatted BOQ item string or None if invalid.
+        """
+        line = line.strip()
+        if not line or '|' not in line:
+            return None
+        
+        parts = [p.strip() for p in line.split('|')]
+        if len(parts) < 7:
+            parts += ['NA'] * (7 - len(parts))
+        
+        # Add source column
+        parts.append("Unknown")
+        
+        # Detect page source
+        desc = parts[1]
+        parts[7] = self._detect_page_source(desc, batch_text)
+        
+        return '|'.join(parts[:8])
+    
+    def _extract_from_batch(self, batch_text: str, batch_num: int) -> List[str]:
+        """
+        Extract BOQ items from a single batch.
+        
+        Args:
+            batch_text: Combined text from batch chunks.
+            batch_num: Batch number for logging.
+        
+        Returns:
+            List of BOQ item strings.
+        """
+        prompt_text = batch_text[:self.max_prompt_length]
+        prompt = BOQ_EXTRACTION_TEMPLATE.format(batch_text=prompt_text)
+        
+        try:
+            logger.info(f'Invoking LLM for BOQ extraction on batch {batch_num}...')
+            result = self.llm_client.invoke(prompt)
+            logger.info(f'LLM response received for batch {batch_num}')
+            
+            if 'NO_BOQ_ITEMS' in result:
+                logger.info(f'No BOQ items found in batch {batch_num}')
+                return []
+            
+            boq_items = []
+            lines = result.strip().split('\n')
+            
+            for line in lines:
+                parsed = self._parse_boq_line(line, batch_text)
+                if parsed:
+                    boq_items.append(parsed)
+            
+            logger.info(f'Extracted {len(boq_items)} BOQ items from batch {batch_num}')
+            return boq_items
+            
+        except Exception as e:
+            logger.warning(f'Error processing batch {batch_num}: {e}')
+            return []
+    
+    def _format_output(self, unique_items: List[str], metadata_result: str) -> str:
+        """
+        Format extracted BOQ items into markdown output.
+        
+        Args:
+            unique_items: List of unique BOQ item strings.
+            metadata_result: Document metadata.
+        
+        Returns:
+            Formatted markdown string.
+        """
+        logger.info('Formatting BOQ output...')
+        
+        if not unique_items:
+            logger.info('No BOQ items to format')
+            return f'''## DOCUMENT SUMMARY
+{metadata_result}
+
+## DETAILED BILL OF QUANTITIES
+No BOQ items were found in this document.'''
+        
+        # Determine which columns have data
+        cols_present = [False] * 8
+        normalized_items = []
+        
+        for item in unique_items:
+            parts = [p.strip() for p in item.split('|')]
+            if len(parts) < 8:
+                parts += ['NA'] * (8 - len(parts))
+            normalized_items.append(parts[:8])
+            
+            for i in range(8):
+                if parts[i] and parts[i].upper() != 'NA':
+                    cols_present[i] = True
+        
+        # Build column indices (always include item code and description)
+        col_indices = [i for i, present in enumerate(cols_present) if present]
+        if 0 not in col_indices:
+            col_indices.insert(0, 0)
+        if 1 not in col_indices:
+            col_indices.insert(1, 1)
+        
+        # Build header and separator rows
+        header_row = '| ' + ' | '.join([BOQ_COLUMN_HEADERS[i] for i in col_indices]) + ' |\n'
+        sep_row = '|' + '|'.join(['-' * (len(BOQ_COLUMN_HEADERS[i]) + 2) for i in col_indices]) + '|\n'
+        
+        formatted_boq = f'''## DOCUMENT SUMMARY
+{metadata_result}
+
+## DETAILED BILL OF QUANTITIES
+**Total Items Found:** {len(unique_items)}
+
+{header_row}{sep_row}'''
+        
+        # Add data rows
+        for parts in normalized_items:
+            # Truncate source if too long
+            parts[7] = parts[7][:self.source_max_length] if len(parts[7]) > self.source_max_length else parts[7]
+            row_vals = [parts[i] for i in col_indices]
+            
+            # Format confidence score
+            if 6 in col_indices:
+                conf_idx = col_indices.index(6)
+                if row_vals[conf_idx] != 'NA':
+                    row_vals[conf_idx] = row_vals[conf_idx].rstrip('%') + '%'
+                    if parts[7] == "Unknown":
+                        row_vals[conf_idx] = "N/A"
+            
+            formatted_boq += '| ' + ' | '.join(row_vals) + ' |\n'
+        
+        # Clean up formatting
+        try:
+            s = formatted_boq.replace('\r\n', '\n').replace('\r', '\n')
+            lines = [ln.lstrip() for ln in s.split('\n')]
+            header_idx = next((i for i, ln in enumerate(lines) if ln.startswith('| ')), None)
+            
+            if header_idx and header_idx > 0 and lines[header_idx - 1].strip():
+                lines.insert(header_idx, '')
+            
+            if header_idx:
+                sep_idx = header_idx + 1
+                if not (sep_idx < len(lines) and re.match(r'^\|\s*-+', lines[sep_idx])):
+                    cols = [c for c in lines[header_idx].split('|') if c.strip()]
+                    sep = '|' + '|'.join(['---' for _ in cols]) + '|'
+                    lines.insert(sep_idx, sep)
+            
+            formatted_boq = '\n'.join(lines).strip() + '\n\n'
+        except Exception:
+            pass
+        
+        return formatted_boq
+    
+    def extract(self, chunks: List[Document], vector_store: FAISS = None) -> str:
+        """
+        Extract BOQ from document chunks.
+        
+        Args:
+            chunks: List of Document chunks.
+            vector_store: Optional vector store (not used currently).
+        
+        Returns:
+            Formatted BOQ output as markdown string.
+        """
+        try:
+            logger.info(f'Starting comprehensive BOQ extraction from {len(chunks)} chunks')
+            
+            # Extract metadata
+            logger.info('Extracting document metadata...')
+            metadata_result = self._extract_metadata(chunks)
+            
+            # Create batches
+            logger.info('Creating batches...')
+            batches = self._batch_chunks(chunks)
+            logger.info(f'Created {len(batches)} batches')
+            
+            # Extract from each batch
+            boq_items = []
+            for batch_num, batch_chunks in enumerate(batches, 1):
+                logger.info(f'Processing batch {batch_num}/{len(batches)} ({len(batch_chunks)} chunks)')
+                
+                chunk_texts = [chunk.page_content for chunk in batch_chunks]
+                batch_text = '\n\n'.join(chunk_texts)
+                logger.info(f'Batch text length: {len(batch_text)}')
+                
+                batch_items = self._extract_from_batch(batch_text, batch_num)
+                boq_items.extend(batch_items)
+                logger.info(f'Batch {batch_num} yielded {len(batch_items)} items')
+            
+            # Deduplicate
+            unique_items = list(dict.fromkeys(boq_items))
+            logger.info(f'Found {len(unique_items)} unique BOQ items after deduplication')
+            
+            # Format output
+            logger.info('Formatting BOQ output...')
+            formatted_boq = self._format_output(unique_items, metadata_result)
+            
+            logger.info('Comprehensive BOQ extraction completed successfully')
+            return formatted_boq
+            
+        except Exception as e:
+            logger.error(f'Error in comprehensive BOQ extraction: {e}')
+            raise

services/consistency.py

@@ -0,0 +1,161 @@
+"""
+Consistency checking service for evaluating BOQ extraction reliability.
+"""
+from typing import List, Dict, Any
+from difflib import SequenceMatcher
+from loguru import logger
+from langchain_core.documents import Document
+from langchain_community.vectorstores import FAISS
+
+from config.settings import settings
+from services.boq_extractor import BOQExtractor
+
+
+class ConsistencyChecker:
+    """
+    Checks consistency of BOQ extractions across multiple runs.
+    
+    Example:
+        checker = ConsistencyChecker()
+        result = checker.check(chunks, vector_store, runs=4)
+        print(f"Consistency: {result['consistency_score']}%")
+    """
+    
+    def __init__(self, boq_extractor: BOQExtractor = None, default_runs: int = None, low_threshold: float = None):
+        """
+        Initialize consistency checker.
+        
+        Args:
+            boq_extractor: BOQ extractor instance. Creates new one if not provided.
+            default_runs: Default number of extraction runs. Defaults to config value.
+            low_threshold: Threshold for low consistency warning. Defaults to config value.
+        """
+        self.boq_extractor = boq_extractor or BOQExtractor()
+        self.default_runs = default_runs or settings.consistency.default_runs
+        self.low_threshold = low_threshold or settings.consistency.low_consistency_threshold
+    
+    def _calculate_similarity(self, results: List[str]) -> float:
+        """
+        Calculate average pairwise similarity between results.
+        
+        Args:
+            results: List of BOQ extraction results.
+        
+        Returns:
+            Average similarity score (0.0 to 1.0).
+        """
+        similarities = []
+        
+        for i in range(len(results)):
+            for j in range(i + 1, len(results)):
+                if results[i] and results[j]:
+                    sim = SequenceMatcher(None, results[i], results[j]).ratio()
+                    similarities.append(sim)
+        
+        return sum(similarities) / len(similarities) if similarities else 0
+    
+    def _extract_confidence_scores(self, boq: str) -> List[float]:
+        """
+        Extract confidence scores from BOQ output.
+        
+        Args:
+            boq: Formatted BOQ output string.
+        
+        Returns:
+            List of confidence score values.
+        """
+        if not boq:
+            return []
+        
+        lines = boq.split('\n')
+        confidence_idx = None
+        confidences = []
+        
+        # Find confidence column index from header
+        for line in lines:
+            line = line.strip()
+            if '|' in line and 'Confidence' in line and not line.startswith('| ---'):
+                parts = [p.strip() for p in line.split('|')[1:-1]]
+                confidence_idx = next(
+                    (i for i, p in enumerate(parts) if 'Confidence' in p),
+                    None
+                )
+                if confidence_idx is not None:
+                    break
+        
+        if confidence_idx is None:
+            return []
+        
+        # Extract confidence values from data rows
+        for line in lines:
+            if '|' in line and not line.startswith('| ---') and 'Confidence' not in line:
+                parts = [p.strip() for p in line.split('|')[1:-1]]
+                if len(parts) > confidence_idx:
+                    try:
+                        conf_str = parts[confidence_idx]
+                        if conf_str and conf_str != 'NA' and conf_str != 'N/A':
+                            conf_str = conf_str.rstrip('%')
+                            conf = float(conf_str)
+                            confidences.append(conf)
+                    except (ValueError, IndexError):
+                        pass
+        
+        return confidences
+    
+    def check(self, chunks: List[Document], vector_store: FAISS, runs: int = None) -> Dict[str, Any]:
+        """
+        Run multiple BOQ extractions and compute consistency metrics.
+        
+        Args:
+            chunks: Document chunks to extract from.
+            vector_store: Vector store (passed to extractor).
+            runs: Number of extraction runs. Defaults to config value.
+        
+        Returns:
+            Dictionary with consistency metrics:
+                - consistency_score: Overall consistency percentage
+                - runs: Number of runs attempted
+                - successful_runs: Number of successful runs
+                - avg_similarity: Average pairwise similarity
+                - avg_confidence: Average confidence score
+                - total_confidence_scores: Number of confidence scores found
+                - is_low_consistency: Whether consistency is below threshold
+        """
+        runs = runs or self.default_runs
+        logger.info(f'Starting consistency check with {runs} runs')
+        
+        results = []
+        for run_num in range(runs):
+            try:
+                logger.info(f'Consistency run {run_num + 1}/{runs}')
+                boq = self.boq_extractor.extract(chunks, vector_store)
+                results.append(boq)
+            except Exception as e:
+                logger.warning(f"Consistency run {run_num + 1} failed: {e}")
+                results.append("")
+        
+        # Calculate similarity
+        avg_similarity = self._calculate_similarity(results)
+        consistency_score = avg_similarity * 100
+        
+        # Extract and average confidence scores
+        all_confidences = []
+        for boq in results:
+            confidences = self._extract_confidence_scores(boq)
+            all_confidences.extend(confidences)
+        
+        avg_confidence = sum(all_confidences) / len(all_confidences) if all_confidences else 0
+        successful_runs = len([r for r in results if r])
+        
+        result = {
+            "consistency_score": round(consistency_score, 2),
+            "runs": runs,
+            "successful_runs": successful_runs,
+            "avg_similarity": round(avg_similarity, 2),
+            "avg_confidence": round(avg_confidence, 2),
+            "total_confidence_scores": len(all_confidences),
+            "is_low_consistency": consistency_score < self.low_threshold
+        }
+        
+        logger.info(f'Consistency check completed: {result}')
+        return result

streamlit_app.py

@@ -1,143 +1,310 @@
-import streamlit as st
+"""
+BOQTenders Streamlit Application
+
+Interactive web interface for BOQ extraction and document chat.
+
+Usage:
+    streamlit run streamlit_app_new.py
+"""
+import sys
+from pathlib import Path
+
+# Add project root to path
+sys.path.insert(0, str(Path(__file__).parent))
+
 import tempfile
-import os
-import boq_processor
-
-st.set_page_config(page_title="BOQ Agent", page_icon="📄", layout="wide")
-
-st.title("BOQ Agent")
-st.markdown("Upload a tender PDF to extract the Bill of Quantities (BOQ) and chat with the document.")
-
-# Initialize session state
-if "qa_chain" not in st.session_state:
-    st.session_state.qa_chain = None
-if "extracted_boq" not in st.session_state:
-    st.session_state.extracted_boq = None
-if "chunks" not in st.session_state:
-    st.session_state.chunks = None
-if "vector_store" not in st.session_state:
-    st.session_state.vector_store = None
-
-if "messages" not in st.session_state:
-    st.session_state.messages = []
-
-# Sidebar
-with st.sidebar:
-    st.header("About")
-    st.markdown("""
-    - Upload a PDF containing BOQ data.
-    - Automatically extract the BOQ.
-    - Ask questions about the document.
-    """)
-
-# File uploader
-uploaded_file = st.file_uploader("Choose a PDF file", type="pdf")
-
-if uploaded_file is not None:
-    if st.button("Generate BOQ"):
-        with st.spinner("Generating BOQ..."):
-            # Save uploaded file to a temp file
-            with tempfile.NamedTemporaryFile(delete=False, suffix=".pdf") as tmp_file:
-                tmp_file.write(uploaded_file.getvalue())
-                tmp_path = tmp_file.name
+import streamlit as st
+from loguru import logger
+
+from config.settings import settings
+from core.pdf_extractor import PDFExtractor
+from core.embeddings import EmbeddingService
+from core.rag_chain import RAGChainBuilder
+from services.boq_extractor import BOQExtractor
+from services.consistency import ConsistencyChecker
+
+# Configure logging
+logger.remove()
+logger.add(
+    sys.stderr,
+    level=settings.log_level,
+    format="<green>{time:YYYY-MM-DD HH:mm:ss}</green> | <level>{level: <8}</level> | <cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>"
+)
+
+
+def initialize_services():
+    """Initialize all services (cached)."""
+    if "services_initialized" not in st.session_state:
+        st.session_state.pdf_extractor = PDFExtractor()
+        st.session_state.embedding_service = EmbeddingService()
+        st.session_state.rag_builder = RAGChainBuilder()
+        st.session_state.boq_extractor = BOQExtractor()
+        st.session_state.consistency_checker = ConsistencyChecker()
+        st.session_state.services_initialized = True
+
+
+def initialize_session_state():
+    """Initialize Streamlit session state variables."""
+    defaults = {
+        "boq_output": None,
+        "qa_chain": None,
+        "vector_store": None,
+        "chunks": None,
+        "chat_history": [],
+        "document_loaded": False,
+    }
+    
+    for key, value in defaults.items():
+        if key not in st.session_state:
+            st.session_state[key] = value
+
+
+def process_pdf(uploaded_file) -> bool:
+    """
+    Process uploaded PDF file.
+    
+    Returns:
+        True if processing succeeded, False otherwise.
+    """
+    try:
+        with st.spinner("Processing PDF..."):
+            # Save to temp file
+            with tempfile.NamedTemporaryFile(delete=False, suffix='.pdf') as temp_file:
+                temp_file.write(uploaded_file.getvalue())
+                temp_path = temp_file.name
             
             try:
-                # Process
-                chunks = boq_processor.load_and_process_pdf(tmp_path, filename=uploaded_file.name)
-                vector_store = boq_processor.create_vector_store(chunks)
-                qa_chain = boq_processor.setup_rag_chain(vector_store)
+                # Extract text
+                st.info("Extracting text from PDF...")
+                text = st.session_state.pdf_extractor.extract_text(temp_path, filename=uploaded_file.name)
+                
+                if not text:
+                    st.error("Could not extract text from PDF")
+                    return False
+                
+                # Create embeddings
+                st.info("Creating embeddings...")
+                chunks = st.session_state.embedding_service.split_text(text)
+                vector_store = st.session_state.embedding_service.create_vector_store(chunks)
                 
-                # Use comprehensive extraction for complete BOQ coverage
-                extracted_boq = boq_processor.extract_boq_comprehensive(chunks, vector_store)
+                # Extract BOQ
+                st.info("Extracting BOQ items...")
+                boq_output = st.session_state.boq_extractor.extract(chunks, vector_store)
+                
+                # Build QA chain
+                st.info("Building chat interface...")
+                qa_chain = st.session_state.rag_builder.build(vector_store)
                 
                 # Store in session
-                st.session_state.qa_chain = qa_chain
-                st.session_state.extracted_boq = extracted_boq
                 st.session_state.chunks = chunks
                 st.session_state.vector_store = vector_store
-                st.session_state.messages = []  # reset chat history
-
-                st.success("✅ BOQ generated successfully!")
-            except Exception as e:
-                error_msg = str(e)
-                if "429" in error_msg or "RESOURCE_EXHAUSTED" in error_msg or "quota" in error_msg.lower():
-                    st.error(f"⚠️ API Rate Limit Exceeded: You've hit the daily quota limit. Please try again later or upgrade your API plan. See https://ai.google.dev/gemini-api/docs/rate-limits")
-                else:
-                    st.error(f"Error processing PDF: {error_msg}")
+                st.session_state.boq_output = boq_output
+                st.session_state.qa_chain = qa_chain
+                st.session_state.document_loaded = True
+                st.session_state.chat_history = []
+                
+                st.success(f"✅ Processed {len(chunks)} document chunks")
+                return True
+                
             finally:
-                os.unlink(tmp_path)
+                Path(temp_path).unlink(missing_ok=True)
+                
+    except Exception as e:
+        logger.error(f"Error processing PDF: {e}")
+        st.error(f"Error processing PDF: {str(e)}")
+        return False
 
-# Display extracted BOQ
-if st.session_state.extracted_boq:
-    st.subheader("📊 Extracted Bill of Quantities")
-    
-    # Parse and display the extracted BOQ with better formatting
-    boq_text = st.session_state.extracted_boq
+
+def render_chat_interface():
+    """Render the chat interface."""
+    st.subheader("💬 Chat with Document")
     
-    # Display as markdown for better rendering
-    st.markdown(boq_text)
+    if not st.session_state.document_loaded:
+        st.info("Please upload a PDF to enable chat")
+        return
     
-    st.divider()
+    # Chat history
+    for message in st.session_state.chat_history:
+        role = message["role"]
+        content = message["content"]
+        
+        if role == "user":
+            st.chat_message("user").write(content)
+        else:
+            st.chat_message("assistant").write(content)
     
-    # Add a download button for the BOQ below
-    st.download_button(
-        label="📥 Download BOQ as Text",
-        data=boq_text,
-        file_name="BOQ_extracted.txt",
-        mime="text/plain"
-    )
-
-    # Add consistency check button
-    if st.button("🔍 Check BOQ Reliability"):
-        with st.spinner("Running consistency check..."):
+    # Chat input
+    if prompt := st.chat_input("Ask a question about the document..."):
+        st.session_state.chat_history.append({"role": "user", "content": prompt})
+        st.chat_message("user").write(prompt)
+        
+        with st.spinner("Thinking..."):
             try:
-                consistency = boq_processor.check_consistency(st.session_state.chunks, st.session_state.vector_store, runs=4)
-                st.success(f"✅ Consistency Check Complete")
-                st.write(f"**Consistency Score:** {consistency['consistency_score']}%")
-                st.write(f"**Successful Runs:** {consistency['successful_runs']}/{consistency['runs']}")
-                st.write(f"**Average Confidence:** {consistency['avg_confidence']:.2f} (from {consistency['total_confidence_scores']} scores)")
-                if consistency['consistency_score'] < 80:
-                    st.warning("⚠️ Low consistency detected. LLM outputs vary significantly—consider reviewing extractions.")
+                response = st.session_state.qa_chain({"question": prompt})
+                answer = response.get("answer", "I couldn't find an answer.")
+                
+                st.session_state.chat_history.append({"role": "assistant", "content": answer})
+                st.chat_message("assistant").write(answer)
+                
             except Exception as e:
-                st.error(f"Consistency check failed: {e}")
+                logger.error(f"Chat error: {e}")
+                error_msg = f"Error: {str(e)}"
+                st.session_state.chat_history.append({"role": "assistant", "content": error_msg})
+                st.chat_message("assistant").write(error_msg)
 
-    st.divider()
 
-    # -------------------------------
-    # Chat Interface
-    # -------------------------------
-    st.subheader("💬 Chat with your BOQ")
+def render_boq_output():
+    """Render the BOQ output."""
+    st.subheader("📋 Extracted BOQ")
+    
+    if st.session_state.boq_output:
+        st.markdown(st.session_state.boq_output)
+        
+        # Download button
+        st.download_button(
+            label="📥 Download BOQ as Markdown",
+            data=st.session_state.boq_output,
+            file_name="boq_output.md",
+            mime="text/markdown"
+        )
+    else:
+        st.info("Upload a PDF to see extracted BOQ items")
 
-    # Display previous chat messages
-    for message in st.session_state.messages:
-        with st.chat_message(message["role"]):
-            st.markdown(message["content"])
 
-    # User input
-    if prompt := st.chat_input("Ask a question about the BOQ"):
-        # Add user message
-        st.session_state.messages.append(
-            {"role": "user", "content": prompt}
-        )
+def render_consistency_check():
+    """Render consistency check interface."""
+    st.subheader("🔍 Consistency Check")
+    
+    if not st.session_state.document_loaded:
+        st.info("Upload a PDF to run consistency checks")
+        return
+    
+    runs = st.number_input(
+        "Number of extraction runs",
+        min_value=2,
+        max_value=10,
+        value=settings.consistency.default_runs,
+        step=1
+    )
+    
+    if st.button("Run Consistency Check"):
+        with st.spinner(f"Running {runs} extraction passes..."):
+            try:
+                result = st.session_state.consistency_checker.check(
+                    chunks=st.session_state.chunks,
+                    vector_store=st.session_state.vector_store,
+                    runs=runs
+                )
+                
+                col1, col2, col3 = st.columns(3)
+                
+                with col1:
+                    st.metric("Consistency Score", f"{result['consistency_score']:.1f}%")
+                
+                with col2:
+                    st.metric("Avg Confidence", f"{result['avg_confidence']:.1f}%")
+                
+                with col3:
+                    st.metric("Successful Runs", f"{result['successful_runs']}/{result['runs']}")
+                
+                if result['is_low_consistency']:
+                    st.warning("⚠️ Low consistency detected. Results may vary.")
+                else:
+                    st.success("✅ Good consistency across extraction runs")
+                    
+            except Exception as e:
+                logger.error(f"Consistency check error: {e}")
+                st.error(f"Error: {str(e)}")
 
-        with st.chat_message("user"):
-            st.markdown(prompt)
 
-        # Assistant response
-        with st.chat_message("assistant"):
-            with st.spinner("Thinking..."):
-                try:
-                    # OLD LangChain API (ConversationalRetrievalChain)
-                    response = st.session_state.qa_chain(
-                        {"question": prompt}
-                    )
+def render_sidebar():
+    """Render the sidebar."""
+    with st.sidebar:
+        st.title("📄 BOQ Extractor")
+        st.markdown("---")
+        
+        # File upload
+        uploaded_file = st.file_uploader(
+            "Upload PDF Document",
+            type=["pdf"],
+            help="Upload a tender/BOQ document for extraction"
+        )
+        
+        if uploaded_file:
+            if st.button("🚀 Process Document"):
+                process_pdf(uploaded_file)
+        
+        st.markdown("---")
+        
+        # Clear session
+        if st.button("🗑️ Clear Session"):
+            for key in list(st.session_state.keys()):
+                if key != "services_initialized":
+                    del st.session_state[key]
+            initialize_session_state()
+            st.success("Session cleared!")
+            st.rerun()
+
 
-                    answer = response.get("answer", "No response generated.")
-                    st.markdown(answer)
+def main():
+    """Main application entry point."""
+    # Page config
+    st.set_page_config(
+        page_title=settings.streamlit.page_title,
+        page_icon=settings.streamlit.page_icon,
+        layout=settings.streamlit.layout,
+        initial_sidebar_state="expanded"
+    )
+    
+    # Add CSS for sticky tabs
+    st.markdown("""
+        <style>
+        /* Make tabs sticky at top */
+        .stTabs [data-baseweb="tab-list"] {
+            position: sticky;
+            top: 0;
+            background-color: white;
+            z-index: 999;
+            padding-top: 1rem;
+            padding-bottom: 0.5rem;
+            border-bottom: 1px solid #e6e6e6;
+        }
+        
+        /* Dark mode support */
+        @media (prefers-color-scheme: dark) {
+            .stTabs [data-baseweb="tab-list"] {
+                background-color: #0e1117;
+                border-bottom: 1px solid #333;
+            }
+        }
+        
+        /* Streamlit dark theme */
+        [data-theme="dark"] .stTabs [data-baseweb="tab-list"] {
+            background-color: #0e1117;
+            border-bottom: 1px solid #333;
+        }
+        </style>
+    """, unsafe_allow_html=True)
+    
+    # Initialize
+    initialize_services()
+    initialize_session_state()
+    
+    # Render sidebar
+    render_sidebar()
+    
+    # Main content tabs
+    tab1, tab2, tab3 = st.tabs(["📋 BOQ Output", "💬 Chat", "🔍 Analysis"])
+    
+    with tab1:
+        render_boq_output()
+    
+    with tab2:
+        render_chat_interface()
+    
+    with tab3:
+        render_consistency_check()
 
-                    st.session_state.messages.append(
-                        {"role": "assistant", "content": answer}
-                    )
 
-                except Exception as e:
-                    st.error(f"Chat error: {e}")
+if __name__ == "__main__":
+    main()