Deploy GraphRAG FastAPI app to Hugging Face Spaces

.dockerignore

@@ -0,0 +1,33 @@
+.git
+.gitignore
+
+venv
+.venv
+env
+
+__pycache__
+*.pyc
+*.pyo
+*.pyd
+
+.env
+.env.*
+*.log
+
+data/uploads
+data/processed
+data/qdrant
+data/evaluation
+
+outputs
+reports
+notebooks
+
+*.pt
+*.pth
+*.bin
+*.safetensors
+*.onnx
+
+.DS_Store
+Thumbs.db

.gitignore

@@ -0,0 +1,30 @@
+venv/
+.venv/
+env/
+
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+
+.env
+.env.*
+*.log
+
+data/uploads/
+data/processed/
+data/qdrant/
+data/evaluation/
+
+outputs/
+reports/
+notebooks/
+
+*.pt
+*.pth
+*.bin
+*.safetensors
+*.onnx
+
+.DS_Store
+Thumbs.db

Dockerfile

@@ -0,0 +1,36 @@
+FROM python:3.11-slim
+
+RUN useradd -m -u 1000 user
+
+ENV HOME=/home/user
+ENV PATH=/home/user/.local/bin:$PATH
+ENV PYTHONUNBUFFERED=1
+ENV PYTHONDONTWRITEBYTECODE=1
+
+WORKDIR $HOME/app
+
+RUN apt-get update && apt-get install -y --no-install-recommends build-essential curl git && rm -rf /var/lib/apt/lists/*
+
+COPY --chown=user requirements.txt $HOME/app/requirements.txt
+
+RUN pip install --no-cache-dir --upgrade pip
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY --chown=user . $HOME/app
+
+USER user
+
+ENV PORT=7860
+ENV LLM_PROVIDER=huggingface
+ENV ENABLE_LOCAL_LLM=false
+ENV HF_INFERENCE_MODEL=google/flan-t5-base
+ENV HF_TIMEOUT_SECONDS=60
+
+ENV UPLOAD_DIR=data/uploads
+ENV PROCESSED_DIR=data/processed
+ENV QDRANT_LOCAL_PATH=data/qdrant
+ENV EVALUATION_DIR=data/evaluation
+
+EXPOSE 7860
+
+CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -0,0 +1,35 @@
+---
+title: GraphRAG Research Scientist
+emoji: 🧠
+colorFrom: indigo
+colorTo: blue
+sdk: docker
+app_port: 7860
+pinned: false
+---
+
+# GraphRAG Research Scientist
+
+A FastAPI-based GraphRAG research assistant for document-grounded question answering.
+
+## Main endpoints
+
+- `/` health check
+- `/demo` simple browser demo
+- `/docs` Swagger API docs
+- `/deployment/health` deployment health
+- `/deployment/config` deployment config
+- `/upload` upload document
+- `/documents/{document_id}/index` index document
+- `/ask` ask question
+
+## Hugging Face Variables
+
+LLM_PROVIDER=huggingface  
+ENABLE_LOCAL_LLM=false  
+HF_INFERENCE_MODEL=google/flan-t5-base  
+HF_TIMEOUT_SECONDS=60  
+
+## Hugging Face Secret
+
+HF_API_TOKEN should be added in Space Settings as a secret.

app/chunking/chunking_service.py

@@ -0,0 +1,100 @@
+from typing import List
+
+from app.core.config import settings
+from app.schemas.rich_content_block import RichContentBlock
+from app.schemas.content_chunk import ContentChunk
+
+
+def chunk_blocks(
+    blocks: List[RichContentBlock],
+    chunk_size: int = settings.DEFAULT_CHUNK_SIZE,
+    chunk_overlap: int = settings.DEFAULT_CHUNK_OVERLAP
+) -> List[ContentChunk]:
+
+    chunks = []
+
+    for block in blocks:
+        if block.content_type == "text":
+            chunks.extend(
+                chunk_text_block(
+                    block=block,
+                    chunk_size=chunk_size,
+                    chunk_overlap=chunk_overlap
+                )
+            )
+        else:
+            chunks.append(create_atomic_chunk(block))
+
+    return chunks
+
+
+def chunk_text_block(
+    block: RichContentBlock,
+    chunk_size: int,
+    chunk_overlap: int
+) -> List[ContentChunk]:
+
+    text = block.content
+
+    if not text:
+        return []
+
+    chunks = []
+    start = 0
+    chunk_index = 0
+    text_length = len(text)
+
+    while start < text_length:
+        end = min(start + chunk_size, text_length)
+        chunk_text = text[start:end].strip()
+
+        if chunk_text:
+            chunks.append(
+                ContentChunk(
+                    chunk_id=f"{block.block_id}_chunk_{chunk_index + 1}",
+                    document_id=block.document_id,
+                    parent_block_id=block.block_id,
+                    content_type=block.content_type,
+                    content=chunk_text,
+                    chunk_index=chunk_index,
+                    page_number=block.page_number,
+                    section_title=block.section_title,
+                    source_file_name=block.source_file_name,
+                    start_char=start,
+                    end_char=end,
+                    metadata={
+                        "chunking_strategy": "character_overlap",
+                        "chunk_size": chunk_size,
+                        "chunk_overlap": chunk_overlap,
+                        "parent_parser": block.metadata.get("parser")
+                    }
+                )
+            )
+            chunk_index += 1
+
+        if end == text_length:
+            break
+
+        start = end - chunk_overlap
+
+    return chunks
+
+
+def create_atomic_chunk(block: RichContentBlock) -> ContentChunk:
+    return ContentChunk(
+        chunk_id=f"{block.block_id}_chunk_1",
+        document_id=block.document_id,
+        parent_block_id=block.block_id,
+        content_type=block.content_type,
+        content=block.content,
+        chunk_index=0,
+        page_number=block.page_number,
+        section_title=block.section_title,
+        source_file_name=block.source_file_name,
+        start_char=0,
+        end_char=len(block.content),
+        metadata={
+            "chunking_strategy": "atomic_rich_content_block",
+            "reason": "non-text content should not be split"
+        }
+    )

app/core/config.py

@@ -0,0 +1,138 @@
+import os
+from dataclasses import dataclass
+from pathlib import Path
+
+
+def get_int_env(variable_name: str, default_value: int) -> int:
+    value = os.getenv(variable_name)
+
+    if value is None:
+        return default_value
+
+    try:
+        return int(value)
+    except ValueError:
+        return default_value
+
+
+def get_float_env(variable_name: str, default_value: float) -> float:
+    value = os.getenv(variable_name)
+
+    if value is None:
+        return default_value
+
+    try:
+        return float(value)
+    except ValueError:
+        return default_value
+
+
+def get_bool_env(variable_name: str, default_value: bool) -> bool:
+    value = os.getenv(variable_name)
+
+    if value is None:
+        return default_value
+
+    value = value.lower().strip()
+
+    if value in ["true", "1", "yes", "y"]:
+        return True
+
+    if value in ["false", "0", "no", "n"]:
+        return False
+
+    return default_value
+
+
+@dataclass(frozen=True)
+class Settings:
+    APP_NAME: str = "GraphRAG Research Scientist"
+    APP_VERSION: str = "10.0.0"
+    ENVIRONMENT: str = os.getenv("ENVIRONMENT", "local")
+
+    UPLOAD_DIR: Path = Path(os.getenv("UPLOAD_DIR", "data/uploads"))
+    PROCESSED_DIR: Path = Path(os.getenv("PROCESSED_DIR", "data/processed"))
+    QDRANT_LOCAL_PATH: Path = Path(os.getenv("QDRANT_LOCAL_PATH", "data/qdrant"))
+    EVALUATION_DIR: Path = Path(os.getenv("EVALUATION_DIR", "data/evaluation"))
+
+    DEFAULT_CHUNK_SIZE: int = get_int_env("DEFAULT_CHUNK_SIZE", 1000)
+    DEFAULT_CHUNK_OVERLAP: int = get_int_env("DEFAULT_CHUNK_OVERLAP", 150)
+    MAX_ROWS_PER_TABLE_BLOCK: int = get_int_env("MAX_ROWS_PER_TABLE_BLOCK", 50)
+    MAX_UPLOAD_SIZE_MB: int = get_int_env("MAX_UPLOAD_SIZE_MB", 100)
+
+    EMBEDDING_MODEL_NAME: str = os.getenv(
+        "EMBEDDING_MODEL_NAME",
+        "sentence-transformers/all-MiniLM-L6-v2"
+    )
+    EMBEDDING_DIMENSION: int = get_int_env("EMBEDDING_DIMENSION", 384)
+
+    QDRANT_COLLECTION_NAME: str = os.getenv(
+        "QDRANT_COLLECTION_NAME",
+        "research_chunks"
+    )
+
+    DEFAULT_TOP_K: int = get_int_env("DEFAULT_TOP_K", 5)
+
+    HYBRID_VECTOR_WEIGHT: float = get_float_env("HYBRID_VECTOR_WEIGHT", 0.65)
+    HYBRID_KEYWORD_WEIGHT: float = get_float_env("HYBRID_KEYWORD_WEIGHT", 0.35)
+
+    ENABLE_RERANKER: bool = get_bool_env("ENABLE_RERANKER", True)
+    RERANKER_MODEL_NAME: str = os.getenv(
+        "RERANKER_MODEL_NAME",
+        "cross-encoder/ms-marco-MiniLM-L-6-v2"
+    )
+    RERANKER_CANDIDATE_MULTIPLIER: int = get_int_env(
+        "RERANKER_CANDIDATE_MULTIPLIER",
+        4
+    )
+
+    # =====================================================
+    # LLM provider settings
+    # =====================================================
+    ENABLE_LOCAL_LLM: bool = get_bool_env("ENABLE_LOCAL_LLM", True)
+
+    # Supported now:
+    # local
+    # huggingface
+    # disabled
+    #
+    # Future:
+    # aws_bedrock
+    # openai
+    LLM_PROVIDER: str = os.getenv("LLM_PROVIDER", "local")
+
+    LOCAL_LLM_MODEL_NAME: str = os.getenv(
+        "LOCAL_LLM_MODEL_NAME",
+        "google/flan-t5-base"
+    )
+
+    LOCAL_LLM_DEVICE: str = os.getenv("LOCAL_LLM_DEVICE", "cpu")
+
+    HF_API_TOKEN: str = os.getenv("HF_API_TOKEN", "")
+    HF_INFERENCE_MODEL: str = os.getenv(
+        "HF_INFERENCE_MODEL",
+        "google/flan-t5-base"
+    )
+    HF_INFERENCE_URL: str = os.getenv(
+        "HF_INFERENCE_URL",
+        ""
+    )
+    HF_TIMEOUT_SECONDS: int = get_int_env("HF_TIMEOUT_SECONDS", 60)
+
+    MAX_GENERATION_TOKENS: int = get_int_env("MAX_GENERATION_TOKENS", 220)
+    LOCAL_LLM_MAX_INPUT_TOKENS: int = get_int_env("LOCAL_LLM_MAX_INPUT_TOKENS", 1024)
+
+    MIN_LLM_ANSWER_WORDS: int = get_int_env("MIN_LLM_ANSWER_WORDS", 20)
+    MAX_CONTEXT_CHARS: int = get_int_env("MAX_CONTEXT_CHARS", 5000)
+
+    ENABLE_STATIC_ASSETS: bool = get_bool_env("ENABLE_STATIC_ASSETS", True)
+
+    def ensure_directories(self) -> None:
+        self.UPLOAD_DIR.mkdir(parents=True, exist_ok=True)
+        self.PROCESSED_DIR.mkdir(parents=True, exist_ok=True)
+        self.QDRANT_LOCAL_PATH.mkdir(parents=True, exist_ok=True)
+        self.EVALUATION_DIR.mkdir(parents=True, exist_ok=True)
+
+
+settings = Settings()
+settings.ensure_directories()

app/deployment/hf_status.py

@@ -0,0 +1,54 @@
+import os
+from typing import Dict, Any
+
+from app.core.config import settings
+
+
+def get_deployment_health() -> Dict[str, Any]:
+    return {
+        "status": "healthy",
+        "service": settings.APP_NAME,
+        "version": settings.APP_VERSION,
+        "environment": settings.ENVIRONMENT,
+        "deployment_target": "hugging_face_spaces",
+        "port": os.getenv("PORT", "7860"),
+        "message": "FastAPI app is running and ready for Hugging Face Spaces."
+    }
+
+
+def get_deployment_config() -> Dict[str, Any]:
+    return {
+        "deployment_target": "hugging_face_spaces",
+        "llm_provider": settings.LLM_PROVIDER,
+        "local_llm_enabled": settings.ENABLE_LOCAL_LLM,
+        "hf_model": settings.HF_INFERENCE_MODEL,
+        "hf_token_present": bool(settings.HF_API_TOKEN),
+        "upload_dir": str(settings.UPLOAD_DIR),
+        "processed_dir": str(settings.PROCESSED_DIR),
+        "qdrant_path": str(settings.QDRANT_LOCAL_PATH),
+        "evaluation_dir": str(settings.EVALUATION_DIR),
+        "reranker_enabled": settings.ENABLE_RERANKER,
+        "storage_warning": "Local Space storage can reset after restart unless persistent storage is attached."
+    }
+
+
+def get_demo_html() -> str:
+    return """
+<!DOCTYPE html>
+<html>
+<head>
+    <title>GraphRAG Research Scientist</title>
+</head>
+<body style="font-family: Arial; max-width: 900px; margin: 40px auto; line-height: 1.6;">
+    <h1>🧠 GraphRAG Research Scientist</h1>
+    <p>FastAPI backend is running.</p>
+    <h2>Useful links</h2>
+    <ul>
+        <li><a href="/docs">Swagger API Docs</a></li>
+        <li><a href="/deployment/health">Deployment Health</a></li>
+        <li><a href="/deployment/config">Deployment Config</a></li>
+        <li><a href="/llm/status">LLM Status</a></li>
+    </ul>
+</body>
+</html>
+"""

app/evaluation/answer_eval_storage.py

@@ -0,0 +1,92 @@
+import json
+import uuid
+from typing import List, Optional
+
+from app.core.config import settings
+from app.schemas.evaluation_schema import (
+    AnswerTestCase,
+    AnswerTestCaseCreate
+)
+
+
+ANSWER_TEST_CASES_PATH = settings.EVALUATION_DIR / "answer_test_cases.json"
+
+
+def load_answer_test_cases() -> List[AnswerTestCase]:
+    settings.EVALUATION_DIR.mkdir(parents=True, exist_ok=True)
+
+    if not ANSWER_TEST_CASES_PATH.exists():
+        return []
+
+    with open(ANSWER_TEST_CASES_PATH, "r", encoding="utf-8") as f:
+        data = json.load(f)
+
+    return [AnswerTestCase(**item) for item in data]
+
+
+def save_answer_test_cases(test_cases: List[AnswerTestCase]) -> None:
+    settings.EVALUATION_DIR.mkdir(parents=True, exist_ok=True)
+
+    with open(ANSWER_TEST_CASES_PATH, "w", encoding="utf-8") as f:
+        json.dump(
+            [test_case.model_dump() for test_case in test_cases],
+            f,
+            indent=2,
+            ensure_ascii=False
+        )
+
+
+def add_answer_test_case(
+    test_case_create: AnswerTestCaseCreate
+) -> AnswerTestCase:
+
+    test_cases = load_answer_test_cases()
+
+    new_test_case = AnswerTestCase(
+        test_case_id=str(uuid.uuid4()),
+        question=test_case_create.question,
+        document_id=test_case_create.document_id,
+        top_k=test_case_create.top_k,
+        retrieval_mode=test_case_create.retrieval_mode,
+        use_reranker=test_case_create.use_reranker,
+        use_llm=test_case_create.use_llm,
+        expected_answer_keywords=test_case_create.expected_answer_keywords,
+        forbidden_answer_keywords=test_case_create.forbidden_answer_keywords,
+        require_citations=test_case_create.require_citations,
+        require_sources=test_case_create.require_sources,
+        minimum_answer_words=test_case_create.minimum_answer_words,
+        minimum_keyword_match_ratio=test_case_create.minimum_keyword_match_ratio,
+        minimum_groundedness_score=test_case_create.minimum_groundedness_score,
+        notes=test_case_create.notes,
+        tags=test_case_create.tags
+    )
+
+    test_cases.append(new_test_case)
+    save_answer_test_cases(test_cases)
+
+    return new_test_case
+
+
+def delete_answer_test_case(test_case_id: str) -> bool:
+    test_cases = load_answer_test_cases()
+
+    remaining_cases = [
+        test_case for test_case in test_cases
+        if test_case.test_case_id != test_case_id
+    ]
+
+    if len(remaining_cases) == len(test_cases):
+        return False
+
+    save_answer_test_cases(remaining_cases)
+    return True
+
+
+def get_answer_test_case(test_case_id: str) -> Optional[AnswerTestCase]:
+    test_cases = load_answer_test_cases()
+
+    for test_case in test_cases:
+        if test_case.test_case_id == test_case_id:
+            return test_case
+
+    return None

app/evaluation/answer_evaluator.py

@@ -0,0 +1,367 @@
+import re
+from typing import List, Optional, Dict, Any
+
+from app.schemas.evaluation_schema import (
+    AnswerTestCase,
+    AnswerEvaluationRunRequest,
+    AnswerSingleResult,
+    AnswerEvaluationSummary,
+    AnswerEvaluationReport
+)
+from app.evaluation.answer_eval_storage import load_answer_test_cases
+from app.generation.answer_service import answer_question
+
+
+STOPWORDS = {
+    "the", "a", "an", "and", "or", "of", "to", "in", "on", "by", "for",
+    "with", "from", "is", "are", "was", "were", "be", "been", "it",
+    "this", "that", "as", "at", "which", "what", "how", "why"
+}
+
+
+def run_answer_evaluation(
+    request: AnswerEvaluationRunRequest
+) -> AnswerEvaluationReport:
+
+    all_test_cases = load_answer_test_cases()
+
+    if request.test_case_ids:
+        selected_ids = set(request.test_case_ids)
+        test_cases = [
+            test_case for test_case in all_test_cases
+            if test_case.test_case_id in selected_ids
+        ]
+    else:
+        test_cases = all_test_cases
+
+    results = []
+
+    for test_case in test_cases:
+        result = evaluate_single_answer_test_case(
+            test_case=test_case,
+            use_llm_override=request.use_llm_override,
+            retrieval_mode_override=request.retrieval_mode_override
+        )
+        results.append(result)
+
+    summary = build_answer_evaluation_summary(results)
+
+    return AnswerEvaluationReport(
+        summary=summary,
+        results=results
+    )
+
+
+def evaluate_single_answer_test_case(
+    test_case: AnswerTestCase,
+    use_llm_override: Optional[bool] = None,
+    retrieval_mode_override: Optional[str] = None
+) -> AnswerSingleResult:
+
+    use_llm = (
+        use_llm_override
+        if use_llm_override is not None
+        else test_case.use_llm
+    )
+
+    retrieval_mode = retrieval_mode_override or test_case.retrieval_mode
+
+    answer_output = answer_question(
+        query=test_case.question,
+        document_id=test_case.document_id,
+        top_k=test_case.top_k,
+        retrieval_mode=retrieval_mode,
+        use_reranker=test_case.use_reranker,
+        use_llm=use_llm
+    )
+
+    answer = answer_output.get("answer", "")
+    citations = answer_output.get("citations", [])
+    sources = answer_output.get("sources", [])
+
+    answer_word_count = count_words(answer)
+    citation_present = has_citation(answer)
+    source_count = len(sources)
+
+    matched_keywords, missing_keywords, keyword_match_ratio = evaluate_keywords(
+        answer=answer,
+        expected_keywords=test_case.expected_answer_keywords
+    )
+
+    forbidden_keywords_found = find_forbidden_keywords(
+        answer=answer,
+        forbidden_keywords=test_case.forbidden_answer_keywords
+    )
+
+    groundedness_score = compute_groundedness_score(
+        answer=answer,
+        sources=sources
+    )
+
+    groundedness_passed = (
+        groundedness_score >= test_case.minimum_groundedness_score
+    )
+
+    failure_reasons = []
+
+    if answer_word_count < test_case.minimum_answer_words:
+        failure_reasons.append(
+            f"Answer is too short. Expected at least {test_case.minimum_answer_words} words."
+        )
+
+    if test_case.require_citations and not citation_present:
+        failure_reasons.append("Answer does not contain required citations.")
+
+    if test_case.require_sources and source_count == 0:
+        failure_reasons.append("Answer does not include any retrieved sources.")
+
+    if test_case.expected_answer_keywords:
+        if keyword_match_ratio < test_case.minimum_keyword_match_ratio:
+            failure_reasons.append(
+                "Answer did not match enough expected keywords."
+            )
+
+    if forbidden_keywords_found:
+        failure_reasons.append(
+            "Answer contains forbidden keywords."
+        )
+
+    if not groundedness_passed:
+        failure_reasons.append(
+            "Answer does not appear grounded enough in retrieved sources."
+        )
+
+    passed = len(failure_reasons) == 0
+
+    return AnswerSingleResult(
+        test_case_id=test_case.test_case_id,
+        question=test_case.question,
+        passed=passed,
+        failure_reasons=failure_reasons,
+        answer=answer,
+        answer_strategy=answer_output.get("answer_strategy"),
+        used_llm=answer_output.get("used_llm", False),
+        used_reranker=answer_output.get("used_reranker", False),
+        retrieval_mode=answer_output.get("retrieval_mode", retrieval_mode),
+        answer_word_count=answer_word_count,
+        citation_present=citation_present,
+        source_count=source_count,
+        keyword_match_ratio=keyword_match_ratio,
+        matched_keywords=matched_keywords,
+        missing_keywords=missing_keywords,
+        forbidden_keywords_found=forbidden_keywords_found,
+        groundedness_score=groundedness_score,
+        groundedness_passed=groundedness_passed,
+        citations_preview=simplify_citations(citations),
+        sources_preview=simplify_sources(sources)
+    )
+
+
+def count_words(text: str) -> int:
+    return len(re.findall(r"[a-zA-Z0-9_]+", text or ""))
+
+
+def has_citation(text: str) -> bool:
+    if not text:
+        return False
+
+    return bool(re.search(r"\[S\d+\]", text))
+
+
+def evaluate_keywords(
+    answer: str,
+    expected_keywords: List[str]
+):
+    if not expected_keywords:
+        return [], [], None
+
+    answer_lower = answer.lower()
+
+    matched_keywords = []
+    missing_keywords = []
+
+    for keyword in expected_keywords:
+        keyword_lower = keyword.lower().strip()
+
+        if keyword_lower in answer_lower:
+            matched_keywords.append(keyword)
+        else:
+            missing_keywords.append(keyword)
+
+    keyword_match_ratio = round(
+        len(matched_keywords) / len(expected_keywords),
+        4
+    )
+
+    return matched_keywords, missing_keywords, keyword_match_ratio
+
+
+def find_forbidden_keywords(
+    answer: str,
+    forbidden_keywords: List[str]
+) -> List[str]:
+
+    if not forbidden_keywords:
+        return []
+
+    answer_lower = answer.lower()
+    found = []
+
+    for keyword in forbidden_keywords:
+        keyword_lower = keyword.lower().strip()
+
+        if keyword_lower in answer_lower:
+            found.append(keyword)
+
+    return found
+
+
+def tokenize_for_groundedness(text: str) -> set:
+    words = re.findall(r"[a-zA-Z0-9_]+", (text or "").lower())
+
+    tokens = {
+        word for word in words
+        if word not in STOPWORDS and len(word) > 2
+    }
+
+    return tokens
+
+
+def compute_groundedness_score(
+    answer: str,
+    sources: List[Dict[str, Any]]
+) -> float:
+
+    answer_tokens = tokenize_for_groundedness(answer)
+
+    if not answer_tokens:
+        return 0.0
+
+    source_text = " ".join(
+        source.get("content", "")
+        for source in sources
+    )
+
+    source_tokens = tokenize_for_groundedness(source_text)
+
+    if not source_tokens:
+        return 0.0
+
+    overlap = answer_tokens.intersection(source_tokens)
+
+    score = len(overlap) / len(answer_tokens)
+
+    return round(score, 4)
+
+
+def simplify_citations(citations: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    simplified = []
+
+    for citation in citations[:5]:
+        simplified.append(
+            {
+                "source_id": citation.get("source_id"),
+                "source_file_name": citation.get("source_file_name"),
+                "page_number": citation.get("page_number"),
+                "citation_text": citation.get("citation_text")
+            }
+        )
+
+    return simplified
+
+
+def simplify_sources(sources: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    simplified = []
+
+    for source in sources[:5]:
+        content = source.get("content", "")
+
+        simplified.append(
+            {
+                "source_id": source.get("source_id"),
+                "score": source.get("score"),
+                "chunk_id": source.get("chunk_id"),
+                "source_file_name": source.get("source_file_name"),
+                "page_number": source.get("page_number"),
+                "content_preview": content[:250]
+            }
+        )
+
+    return simplified
+
+
+def build_answer_evaluation_summary(
+    results: List[AnswerSingleResult]
+) -> AnswerEvaluationSummary:
+
+    total_cases = len(results)
+
+    if total_cases == 0:
+        return AnswerEvaluationSummary(
+            total_cases=0,
+            passed_cases=0,
+            failed_cases=0,
+            pass_rate=0.0,
+            average_groundedness_score=0.0,
+            average_answer_word_count=0.0
+        )
+
+    passed_cases = sum(1 for result in results if result.passed)
+    failed_cases = total_cases - passed_cases
+
+    pass_rate = round(passed_cases / total_cases, 4)
+
+    citation_pass_rate = round(
+        sum(1 for result in results if result.citation_present) / total_cases,
+        4
+    )
+
+    source_presence_rate = round(
+        sum(1 for result in results if result.source_count > 0) / total_cases,
+        4
+    )
+
+    keyword_results = [
+        result for result in results
+        if result.keyword_match_ratio is not None
+    ]
+
+    keyword_pass_rate = None
+
+    if keyword_results:
+        keyword_pass_rate = round(
+            sum(
+                1 for result in keyword_results
+                if result.keyword_match_ratio is not None
+                and result.keyword_match_ratio >= 0.5
+            ) / len(keyword_results),
+            4
+        )
+
+    groundedness_pass_rate = round(
+        sum(1 for result in results if result.groundedness_passed) / total_cases,
+        4
+    )
+
+    average_groundedness_score = round(
+        sum(result.groundedness_score for result in results) / total_cases,
+        4
+    )
+
+    average_answer_word_count = round(
+        sum(result.answer_word_count for result in results) / total_cases,
+        2
+    )
+
+    return AnswerEvaluationSummary(
+        total_cases=total_cases,
+        passed_cases=passed_cases,
+        failed_cases=failed_cases,
+        pass_rate=pass_rate,
+        citation_pass_rate=citation_pass_rate,
+        source_presence_rate=source_presence_rate,
+        keyword_pass_rate=keyword_pass_rate,
+        groundedness_pass_rate=groundedness_pass_rate,
+        average_groundedness_score=average_groundedness_score,
+        average_answer_word_count=average_answer_word_count
+    )

app/evaluation/retrieval_eval_storage.py

@@ -0,0 +1,87 @@
+import json
+import uuid
+from typing import List, Optional
+
+from app.core.config import settings
+from app.schemas.evaluation_schema import (
+    RetrievalTestCase,
+    RetrievalTestCaseCreate
+)
+
+
+TEST_CASES_PATH = settings.EVALUATION_DIR / "retrieval_test_cases.json"
+
+
+def load_retrieval_test_cases() -> List[RetrievalTestCase]:
+    settings.EVALUATION_DIR.mkdir(parents=True, exist_ok=True)
+
+    if not TEST_CASES_PATH.exists():
+        return []
+
+    with open(TEST_CASES_PATH, "r", encoding="utf-8") as f:
+        data = json.load(f)
+
+    return [RetrievalTestCase(**item) for item in data]
+
+
+def save_retrieval_test_cases(test_cases: List[RetrievalTestCase]) -> None:
+    settings.EVALUATION_DIR.mkdir(parents=True, exist_ok=True)
+
+    with open(TEST_CASES_PATH, "w", encoding="utf-8") as f:
+        json.dump(
+            [test_case.model_dump() for test_case in test_cases],
+            f,
+            indent=2,
+            ensure_ascii=False
+        )
+
+
+def add_retrieval_test_case(
+    test_case_create: RetrievalTestCaseCreate
+) -> RetrievalTestCase:
+
+    test_cases = load_retrieval_test_cases()
+
+    new_test_case = RetrievalTestCase(
+        test_case_id=str(uuid.uuid4()),
+        question=test_case_create.question,
+        expected_document_id=test_case_create.expected_document_id,
+        expected_source_file_name=test_case_create.expected_source_file_name,
+        expected_page_numbers=test_case_create.expected_page_numbers,
+        expected_chunk_ids=test_case_create.expected_chunk_ids,
+        search_document_id=test_case_create.search_document_id,
+        top_k=test_case_create.top_k,
+        retrieval_mode=test_case_create.retrieval_mode,
+        notes=test_case_create.notes,
+        tags=test_case_create.tags
+    )
+
+    test_cases.append(new_test_case)
+    save_retrieval_test_cases(test_cases)
+
+    return new_test_case
+
+
+def delete_retrieval_test_case(test_case_id: str) -> bool:
+    test_cases = load_retrieval_test_cases()
+
+    remaining_cases = [
+        test_case for test_case in test_cases
+        if test_case.test_case_id != test_case_id
+    ]
+
+    if len(remaining_cases) == len(test_cases):
+        return False
+
+    save_retrieval_test_cases(remaining_cases)
+    return True
+
+
+def get_retrieval_test_case(test_case_id: str) -> Optional[RetrievalTestCase]:
+    test_cases = load_retrieval_test_cases()
+
+    for test_case in test_cases:
+        if test_case.test_case_id == test_case_id:
+            return test_case
+
+    return None

app/evaluation/retrieval_evaluator.py

@@ -0,0 +1,345 @@
+from typing import List, Optional, Dict, Any
+
+from app.schemas.evaluation_schema import (
+    RetrievalTestCase,
+    RetrievalEvaluationRunRequest,
+    RetrievalSingleResult,
+    RetrievalEvaluationSummary,
+    RetrievalEvaluationReport
+)
+from app.evaluation.retrieval_eval_storage import load_retrieval_test_cases
+from app.retrieval.hybrid_search_service import retrieve_chunks
+
+
+def run_retrieval_evaluation(
+    request: RetrievalEvaluationRunRequest
+) -> RetrievalEvaluationReport:
+
+    all_test_cases = load_retrieval_test_cases()
+
+    if request.test_case_ids:
+        selected_ids = set(request.test_case_ids)
+        test_cases = [
+            test_case for test_case in all_test_cases
+            if test_case.test_case_id in selected_ids
+        ]
+    else:
+        test_cases = all_test_cases
+
+    results = []
+
+    for test_case in test_cases:
+        result = evaluate_single_test_case(
+            test_case=test_case,
+            top_k_override=request.top_k_override,
+            retrieval_mode_override=request.retrieval_mode_override
+        )
+        results.append(result)
+
+    summary = build_evaluation_summary(results)
+
+    return RetrievalEvaluationReport(
+        summary=summary,
+        results=results
+    )
+
+
+def evaluate_single_test_case(
+    test_case: RetrievalTestCase,
+    top_k_override: Optional[int] = None,
+    retrieval_mode_override: Optional[str] = None
+) -> RetrievalSingleResult:
+
+    top_k = top_k_override or test_case.top_k
+    retrieval_mode = retrieval_mode_override or test_case.retrieval_mode
+
+    retrieval_output = retrieve_chunks(
+        query=test_case.question,
+        document_id=test_case.search_document_id,
+        top_k=top_k,
+        retrieval_mode=retrieval_mode
+    )
+
+    retrieved_results = retrieval_output.get("results", [])
+
+    expected_document_hit = evaluate_expected_document_hit(
+        retrieved_results,
+        test_case.expected_document_id
+    )
+
+    expected_source_file_hit = evaluate_expected_source_file_hit(
+        retrieved_results,
+        test_case.expected_source_file_name
+    )
+
+    expected_page_hit = evaluate_expected_page_hit(
+        retrieved_results,
+        test_case.expected_page_numbers
+    )
+
+    expected_chunk_hit = evaluate_expected_chunk_hit(
+        retrieved_results,
+        test_case.expected_chunk_ids
+    )
+
+    best_match_rank = find_best_match_rank(
+        retrieved_results=retrieved_results,
+        test_case=test_case
+    )
+
+    reciprocal_rank = 0.0
+
+    if best_match_rank is not None and best_match_rank > 0:
+        reciprocal_rank = 1.0 / best_match_rank
+
+    failure_reasons = build_failure_reasons(
+        expected_document_hit=expected_document_hit,
+        expected_source_file_hit=expected_source_file_hit,
+        expected_page_hit=expected_page_hit,
+        expected_chunk_hit=expected_chunk_hit
+    )
+
+    passed = len(failure_reasons) == 0
+
+    top_result = None
+
+    if retrieved_results:
+        top_result = simplify_result(retrieved_results[0], rank=1)
+
+    retrieved_results_preview = [
+        simplify_result(result, rank=index + 1)
+        for index, result in enumerate(retrieved_results[:10])
+    ]
+
+    return RetrievalSingleResult(
+        test_case_id=test_case.test_case_id,
+        question=test_case.question,
+        passed=passed,
+        failure_reasons=failure_reasons,
+        expected_document_id=test_case.expected_document_id,
+        expected_source_file_name=test_case.expected_source_file_name,
+        expected_page_numbers=test_case.expected_page_numbers,
+        expected_chunk_ids=test_case.expected_chunk_ids,
+        top_k=top_k,
+        retrieval_mode=retrieval_mode,
+        retrieved_count=len(retrieved_results),
+        expected_document_hit=expected_document_hit,
+        expected_source_file_hit=expected_source_file_hit,
+        expected_page_hit=expected_page_hit,
+        expected_chunk_hit=expected_chunk_hit,
+        best_match_rank=best_match_rank,
+        reciprocal_rank=reciprocal_rank,
+        top_result=top_result,
+        retrieved_results_preview=retrieved_results_preview
+    )
+
+
+def evaluate_expected_document_hit(
+    results: List[Dict[str, Any]],
+    expected_document_id: Optional[str]
+) -> Optional[bool]:
+
+    if not expected_document_id:
+        return None
+
+    return any(
+        result.get("document_id") == expected_document_id
+        for result in results
+    )
+
+
+def evaluate_expected_source_file_hit(
+    results: List[Dict[str, Any]],
+    expected_source_file_name: Optional[str]
+) -> Optional[bool]:
+
+    if not expected_source_file_name:
+        return None
+
+    return any(
+        result.get("source_file_name") == expected_source_file_name
+        for result in results
+    )
+
+
+def evaluate_expected_page_hit(
+    results: List[Dict[str, Any]],
+    expected_page_numbers: List[int]
+) -> Optional[bool]:
+
+    if not expected_page_numbers:
+        return None
+
+    expected_pages = set(expected_page_numbers)
+
+    return any(
+        result.get("page_number") in expected_pages
+        for result in results
+    )
+
+
+def evaluate_expected_chunk_hit(
+    results: List[Dict[str, Any]],
+    expected_chunk_ids: List[str]
+) -> Optional[bool]:
+
+    if not expected_chunk_ids:
+        return None
+
+    expected_chunks = set(expected_chunk_ids)
+
+    return any(
+        result.get("chunk_id") in expected_chunks
+        for result in results
+    )
+
+
+def find_best_match_rank(
+    retrieved_results: List[Dict[str, Any]],
+    test_case: RetrievalTestCase
+) -> Optional[int]:
+
+    for index, result in enumerate(retrieved_results, start=1):
+        if result_matches_any_expectation(result, test_case):
+            return index
+
+    return None
+
+
+def result_matches_any_expectation(
+    result: Dict[str, Any],
+    test_case: RetrievalTestCase
+) -> bool:
+
+    if (
+        test_case.expected_chunk_ids
+        and result.get("chunk_id") in set(test_case.expected_chunk_ids)
+    ):
+        return True
+
+    if (
+        test_case.expected_page_numbers
+        and result.get("page_number") in set(test_case.expected_page_numbers)
+    ):
+        return True
+
+    if (
+        test_case.expected_document_id
+        and result.get("document_id") == test_case.expected_document_id
+    ):
+        return True
+
+    if (
+        test_case.expected_source_file_name
+        and result.get("source_file_name") == test_case.expected_source_file_name
+    ):
+        return True
+
+    return False
+
+
+def build_failure_reasons(
+    expected_document_hit: Optional[bool],
+    expected_source_file_hit: Optional[bool],
+    expected_page_hit: Optional[bool],
+    expected_chunk_hit: Optional[bool]
+) -> List[str]:
+
+    failure_reasons = []
+
+    if expected_document_hit is False:
+        failure_reasons.append("Expected document was not retrieved.")
+
+    if expected_source_file_hit is False:
+        failure_reasons.append("Expected source file was not retrieved.")
+
+    if expected_page_hit is False:
+        failure_reasons.append("Expected page was not retrieved.")
+
+    if expected_chunk_hit is False:
+        failure_reasons.append("Expected chunk was not retrieved.")
+
+    return failure_reasons
+
+
+def simplify_result(result: Dict[str, Any], rank: int) -> Dict[str, Any]:
+    content = result.get("content", "")
+
+    return {
+        "rank": rank,
+        "score": result.get("score"),
+        "chunk_id": result.get("chunk_id"),
+        "document_id": result.get("document_id"),
+        "source_file_name": result.get("source_file_name"),
+        "page_number": result.get("page_number"),
+        "content_type": result.get("content_type"),
+        "content_preview": content[:300]
+    }
+
+
+def build_evaluation_summary(
+    results: List[RetrievalSingleResult]
+) -> RetrievalEvaluationSummary:
+
+    total_cases = len(results)
+
+    if total_cases == 0:
+        return RetrievalEvaluationSummary(
+            total_cases=0,
+            passed_cases=0,
+            failed_cases=0,
+            pass_rate=0.0,
+            mean_reciprocal_rank=0.0
+        )
+
+    passed_cases = sum(1 for result in results if result.passed)
+    failed_cases = total_cases - passed_cases
+
+    pass_rate = round(passed_cases / total_cases, 4)
+
+    mean_reciprocal_rank = round(
+        sum(result.reciprocal_rank for result in results) / total_cases,
+        4
+    )
+
+    document_hit_rate = compute_optional_rate(
+        [result.expected_document_hit for result in results]
+    )
+
+    source_file_hit_rate = compute_optional_rate(
+        [result.expected_source_file_hit for result in results]
+    )
+
+    page_hit_rate = compute_optional_rate(
+        [result.expected_page_hit for result in results]
+    )
+
+    chunk_hit_rate = compute_optional_rate(
+        [result.expected_chunk_hit for result in results]
+    )
+
+    return RetrievalEvaluationSummary(
+        total_cases=total_cases,
+        passed_cases=passed_cases,
+        failed_cases=failed_cases,
+        pass_rate=pass_rate,
+        mean_reciprocal_rank=mean_reciprocal_rank,
+        document_hit_rate=document_hit_rate,
+        source_file_hit_rate=source_file_hit_rate,
+        page_hit_rate=page_hit_rate,
+        chunk_hit_rate=chunk_hit_rate
+    )
+
+
+def compute_optional_rate(values: List[Optional[bool]]) -> Optional[float]:
+    actual_values = [
+        value for value in values
+        if value is not None
+    ]
+
+    if not actual_values:
+        return None
+
+    true_count = sum(1 for value in actual_values if value is True)
+
+    return round(true_count / len(actual_values), 4)

app/generation/answer_quality_checker.py

@@ -0,0 +1,110 @@
+import re
+from typing import List, Dict, Any
+
+from app.core.config import settings
+
+
+BAD_ANSWER_MARKERS = [
+    "local llm generation failed",
+    "i don't know",
+    "i do not know",
+    "unknown",
+    "not enough information",
+    "could not find",
+    "cannot answer",
+    "as an ai",
+    "i am unable",
+    "the evidence does not"
+]
+
+
+def answer_has_citation(answer: str) -> bool:
+    if not answer:
+        return False
+
+    return bool(re.search(r"\[S\d+\]", answer))
+
+
+def answer_is_too_short(answer: str) -> bool:
+    if not answer:
+        return True
+
+    return len(answer.strip().split()) < settings.MIN_LLM_ANSWER_WORDS
+
+
+def answer_repeats_prompt(answer: str) -> bool:
+    answer_lower = answer.lower()
+
+    prompt_markers = [
+        "you are a research assistant",
+        "answer the question using",
+        "sources:",
+        "question:",
+        "evidence:",
+        "final answer:"
+    ]
+
+    return any(marker in answer_lower for marker in prompt_markers)
+
+
+def answer_has_bad_marker(answer: str) -> bool:
+    answer_lower = answer.lower()
+
+    return any(marker in answer_lower for marker in BAD_ANSWER_MARKERS)
+
+
+def answer_is_mostly_citation(answer: str) -> bool:
+    without_citations = re.sub(r"\[S\d+\]", "", answer).strip()
+
+    return len(without_citations.split()) < 8
+
+
+def is_answer_good_enough(answer: str) -> bool:
+    """
+    Quality gate for accepting LLM answer.
+
+    If answer fails this, we use evidence-based fallback.
+    """
+
+    if answer_is_too_short(answer):
+        return False
+
+    if answer_repeats_prompt(answer):
+        return False
+
+    if answer_has_bad_marker(answer):
+        return False
+
+    if answer_is_mostly_citation(answer):
+        return False
+
+    if not answer_has_citation(answer):
+        return False
+
+    return True
+
+
+def append_missing_citations(answer: str, sources: List[Dict[str, Any]]) -> str:
+    """
+    If model gives a good explanation but forgets citations,
+    append top citations. Quality checker still decides acceptance.
+    """
+
+    if not answer:
+        return answer
+
+    if answer_has_citation(answer):
+        return answer
+
+    citation_ids = []
+
+    for source in sources[:2]:
+        source_id = source.get("source_id")
+
+        if source_id:
+            citation_ids.append(f"[{source_id}]")
+
+    if not citation_ids:
+        return answer
+
+    return answer.strip() + " " + " ".join(citation_ids)

app/generation/answer_service.py

@@ -0,0 +1,475 @@
+import re
+from typing import Optional, Dict, Any, List
+
+from app.core.config import settings
+from app.retrieval.hybrid_search_service import retrieve_chunks
+from app.retrieval.reranking_service import rerank_results
+from app.retrieval.citation_service import (
+    attach_source_ids,
+    create_citation_objects
+)
+from app.generation.context_cleaner import clean_retrieved_results, clean_sentence_text
+from app.generation.question_classifier import classify_question
+from app.generation.evidence_extractor import (
+    extract_evidence_sentences,
+    build_evidence_context
+)
+from app.generation.prompt_builder import build_grounded_prompt
+from app.generation.llm_service import generate_with_local_llm, get_llm_status
+from app.generation.answer_quality_checker import (
+    is_answer_good_enough,
+    append_missing_citations
+)
+
+
+def answer_question(
+    query: str,
+    document_id: Optional[str] = None,
+    top_k: int = 5,
+    retrieval_mode: str = "hybrid",
+    use_reranker: bool = True,
+    use_llm: bool = True
+) -> Dict[str, Any]:
+
+    candidate_k = top_k
+
+    if use_reranker:
+        candidate_k = max(
+            top_k * settings.RERANKER_CANDIDATE_MULTIPLIER,
+            top_k
+        )
+
+    retrieval_output = retrieve_chunks(
+        query=query,
+        document_id=document_id,
+        top_k=candidate_k,
+        retrieval_mode=retrieval_mode
+    )
+
+    retrieved_results = retrieval_output["results"]
+
+    if use_reranker:
+        retrieved_results = rerank_results(
+            query=query,
+            results=retrieved_results,
+            top_k=top_k
+        )
+    else:
+        retrieved_results = retrieved_results[:top_k]
+
+    cleaned_results = clean_retrieved_results(retrieved_results)
+    sourced_results = attach_source_ids(cleaned_results)
+
+    citations = create_citation_objects(sourced_results)
+
+    if not sourced_results:
+        return {
+            "query": query,
+            "answer": "I could not find relevant indexed sources for this question.",
+            "retrieval_mode": retrieval_mode,
+            "question_type": classify_question(query),
+            "used_reranker": use_reranker,
+            "used_llm": False,
+            "answer_strategy": "no_sources_found",
+            "citations": [],
+            "sources": []
+        }
+
+    question_type = classify_question(query)
+
+    evidence_items = extract_evidence_sentences(
+        query=query,
+        results=sourced_results,
+        max_evidence=8
+    )
+
+    if not evidence_items:
+        answer = build_extractive_answer(
+            sources=sourced_results
+        )
+
+        return {
+            "query": query,
+            "answer": answer,
+            "retrieval_mode": retrieval_mode,
+            "question_type": question_type,
+            "used_reranker": use_reranker,
+            "used_llm": False,
+            "answer_strategy": "fallback_no_evidence_sentences",
+            "llm_status": get_llm_status(),
+            "citations": citations,
+            "evidence": [],
+            "sources": sourced_results
+        }
+
+    evidence_context = build_evidence_context(evidence_items)
+
+    raw_llm_answer = ""
+    llm_answer_after_citations = ""
+
+    if use_llm:
+        prompt = build_grounded_prompt(
+            query=query,
+            evidence_context=evidence_context,
+            question_type=question_type
+        )
+
+        raw_llm_answer = generate_with_local_llm(prompt)
+
+        llm_answer_after_citations = append_missing_citations(
+            answer=raw_llm_answer,
+            sources=sourced_results
+        )
+
+        if is_answer_good_enough(llm_answer_after_citations):
+            answer = clean_final_answer(llm_answer_after_citations)
+            used_llm = True
+            answer_strategy = "llm_with_quality_check"
+        else:
+            answer = build_evidence_based_answer(
+                query=query,
+                question_type=question_type,
+                evidence_items=evidence_items
+            )
+            used_llm = False
+            answer_strategy = "fallback_evidence_based_answer"
+
+    else:
+        answer = build_evidence_based_answer(
+            query=query,
+            question_type=question_type,
+            evidence_items=evidence_items
+        )
+        used_llm = False
+        answer_strategy = "evidence_based_answer_no_llm"
+
+    answer = clean_final_answer(answer)
+
+    return {
+        "query": query,
+        "answer": answer,
+        "retrieval_mode": retrieval_mode,
+        "question_type": question_type,
+        "used_reranker": use_reranker,
+        "used_llm": used_llm,
+        "answer_strategy": answer_strategy,
+        "llm_status": get_llm_status(),
+        "llm_diagnostics": {
+            "raw_llm_answer_preview": raw_llm_answer[:300],
+            "llm_answer_after_citations_preview": llm_answer_after_citations[:300],
+            "llm_answer_accepted": used_llm
+        },
+        "citations": citations,
+        "evidence": evidence_items,
+        "sources": sourced_results
+    }
+
+
+def build_evidence_based_answer(
+    query: str,
+    question_type: str,
+    evidence_items: List[Dict[str, Any]]
+) -> str:
+
+    if question_type == "definition":
+        return build_definition_answer(query, evidence_items)
+
+    if question_type == "summary":
+        return build_summary_answer(evidence_items)
+
+    if question_type == "comparison":
+        return build_general_answer(evidence_items)
+
+    if question_type == "steps":
+        return build_step_answer(evidence_items)
+
+    return build_general_answer(evidence_items)
+
+
+def build_definition_answer(
+    query: str,
+    evidence_items: List[Dict[str, Any]]
+) -> str:
+
+    target = extract_definition_target(query)
+
+    if target and target.lower() == "rag":
+        return build_rag_definition_answer(evidence_items)
+
+    selected_items = select_best_unique_items(
+        evidence_items=evidence_items,
+        max_items=3
+    )
+
+    lines = []
+
+    for item in selected_items:
+        sentence = clean_sentence_text(item["sentence"])
+        citation = source_id_to_bracket(item.get("source_id"))
+
+        if citation and citation not in sentence:
+            sentence = f"{sentence} {citation}"
+
+        lines.append(sentence)
+
+    return " ".join(lines)
+
+
+def build_rag_definition_answer(evidence_items: List[Dict[str, Any]]) -> str:
+    definition_source = find_first_item_containing(
+        evidence_items,
+        ["retrieval-augmented generation", "retrieval augmented generation"]
+    )
+
+    how_source = find_first_item_containing(
+        evidence_items,
+        [
+            "retrieval step",
+            "before generation",
+            "before generating",
+            "search a document corpus",
+            "search your document corpus",
+            "relevant passages as context"
+        ]
+    )
+
+    why_source = find_first_item_containing(
+        evidence_items,
+        [
+            "frozen knowledge",
+            "hallucination",
+            "private or recent data",
+            "grounds the answer",
+            "real evidence"
+        ]
+    )
+
+    citation_ids = collect_source_ids(
+        [definition_source, how_source, why_source]
+    )
+
+    citation_text = " ".join(
+        source_id_to_bracket(source_id)
+        for source_id in citation_ids
+    )
+
+    answer = (
+        "RAG stands for Retrieval-Augmented Generation. "
+        "It is a method where the system first retrieves relevant passages from a document corpus "
+        "and then provides those passages as context before generating an answer. "
+        "This helps the model answer using real evidence instead of relying only on frozen training knowledge, "
+        "which reduces hallucination and makes the system useful for private or recent information."
+    )
+
+    if citation_text:
+        answer = f"{answer} {citation_text}"
+
+    return answer
+
+
+def build_summary_answer(evidence_items: List[Dict[str, Any]]) -> str:
+    selected_items = select_best_unique_items(
+        evidence_items=evidence_items,
+        max_items=5
+    )
+
+    lines = ["Here is the source-grounded summary:"]
+
+    for index, item in enumerate(selected_items, start=1):
+        sentence = clean_sentence_text(item["sentence"])
+        citation = source_id_to_bracket(item.get("source_id"))
+
+        lines.append(f"{index}. {sentence} {citation}")
+
+    return "\n".join(lines)
+
+
+def build_step_answer(evidence_items: List[Dict[str, Any]]) -> str:
+    selected_items = select_best_unique_items(
+        evidence_items=evidence_items,
+        max_items=5
+    )
+
+    lines = ["Based on the retrieved sources, the process is:"]
+
+    for index, item in enumerate(selected_items, start=1):
+        sentence = clean_sentence_text(item["sentence"])
+        citation = source_id_to_bracket(item.get("source_id"))
+
+        lines.append(f"{index}. {sentence} {citation}")
+
+    return "\n".join(lines)
+
+
+def build_general_answer(evidence_items: List[Dict[str, Any]]) -> str:
+    selected_items = select_best_unique_items(
+        evidence_items=evidence_items,
+        max_items=4
+    )
+
+    lines = []
+
+    for item in selected_items:
+        sentence = clean_sentence_text(item["sentence"])
+        citation = source_id_to_bracket(item.get("source_id"))
+
+        if citation and citation not in sentence:
+            sentence = f"{sentence} {citation}"
+
+        lines.append(sentence)
+
+    return " ".join(lines)
+
+
+def build_extractive_answer(
+    sources: List[Dict[str, Any]]
+) -> str:
+
+    lines = [
+        "I found relevant source-backed passages, but could not extract a cleaner evidence sentence automatically:"
+    ]
+
+    for index, source in enumerate(sources[:3], start=1):
+        content = source.get("content", "")
+        source_id = source.get("source_id", f"S{index}")
+        excerpt = content[:600].replace("\n", " ").strip()
+
+        lines.append(
+            f"{index}. {excerpt} [{source_id}]"
+        )
+
+    return "\n\n".join(lines)
+
+
+def extract_definition_target(query: str) -> Optional[str]:
+    query_lower = query.lower().strip()
+
+    patterns = [
+        r"what is\s+(.+?)\??$",
+        r"what are\s+(.+?)\??$",
+        r"define\s+(.+?)\??$",
+        r"meaning of\s+(.+?)\??$"
+    ]
+
+    for pattern in patterns:
+        match = re.search(pattern, query_lower)
+
+        if match:
+            target = match.group(1).strip()
+            target = target.replace("?", "").strip()
+            return target
+
+    return None
+
+
+def find_first_item_containing(
+    evidence_items: List[Dict[str, Any]],
+    keywords: List[str]
+) -> Optional[Dict[str, Any]]:
+
+    for item in evidence_items:
+        sentence_lower = item.get("sentence", "").lower()
+
+        for keyword in keywords:
+            if keyword.lower() in sentence_lower:
+                return item
+
+    return None
+
+
+def collect_source_ids(items: List[Optional[Dict[str, Any]]]) -> List[str]:
+    source_ids = []
+
+    for item in items:
+        if not item:
+            continue
+
+        source_id = item.get("source_id")
+
+        if source_id and source_id not in source_ids:
+            source_ids.append(source_id)
+
+    return source_ids[:3]
+
+
+def select_best_unique_items(
+    evidence_items: List[Dict[str, Any]],
+    max_items: int
+) -> List[Dict[str, Any]]:
+
+    selected = []
+    seen_meanings = []
+
+    for item in evidence_items:
+        sentence = clean_sentence_text(item["sentence"])
+
+        if is_repetitive_meaning(sentence, seen_meanings):
+            continue
+
+        selected.append(item)
+        seen_meanings.append(sentence)
+
+        if len(selected) >= max_items:
+            break
+
+    return selected
+
+
+def is_repetitive_meaning(sentence: str, existing_sentences: List[str]) -> bool:
+    current_tokens = set(normalize_text(sentence).split())
+
+    if not current_tokens:
+        return True
+
+    for existing in existing_sentences:
+        existing_tokens = set(normalize_text(existing).split())
+
+        if not existing_tokens:
+            continue
+
+        overlap = len(current_tokens.intersection(existing_tokens))
+        union = len(current_tokens.union(existing_tokens))
+
+        if union == 0:
+            continue
+
+        similarity = overlap / union
+
+        if similarity >= 0.65:
+            return True
+
+    return False
+
+
+def normalize_text(text: str) -> str:
+    text = text.lower()
+    text = re.sub(r"[^a-z0-9\s]", " ", text)
+    text = re.sub(r"\b(ideal|answer|question|chapter|page)\b", " ", text)
+    text = re.sub(r"\s+", " ", text)
+    return text.strip()
+
+
+def clean_final_answer(answer: str) -> str:
+    if not answer:
+        return ""
+
+    cleaned = answer
+
+    cleaned = re.sub(r"\bIdeal Answer\b", "", cleaned, flags=re.IGNORECASE)
+    cleaned = re.sub(r"\bQ\d+\s*:\s*", "", cleaned, flags=re.IGNORECASE)
+    cleaned = re.sub(r"\s+", " ", cleaned)
+    cleaned = cleaned.replace(" .", ".")
+    cleaned = cleaned.replace(" ,", ",")
+    cleaned = cleaned.strip()
+
+    return cleaned
+
+
+def source_id_to_bracket(source_id: Optional[str]) -> str:
+    if not source_id:
+        return ""
+
+    if source_id.startswith("[") and source_id.endswith("]"):
+        return source_id
+
+    return f"[{source_id}]"

app/generation/context_cleaner.py

@@ -0,0 +1,72 @@
+import re
+from typing import List, Dict, Any
+
+
+NOISE_PATTERNS = [
+    r"Vectorless RAG Master Guide\s+Vectorless Enterprise Knowledge Intelligence Platform",
+    r"Page\s+\d+\s+of\s+\d+",
+    r"Chapter\s+\d+\s*[:\-].*?(?=\s{2,}|$)",
+    r"Q\d+\s*:\s*",
+    r"Ideal Answer",
+    r"Practice saying these out loud.*?(?=\s{2,}|$)",
+]
+
+
+def clean_chunk_text(text: str) -> str:
+    """
+    Cleans noisy PDF/chunk text before answer generation.
+    """
+
+    if not text:
+        return ""
+
+    cleaned = text
+
+    for pattern in NOISE_PATTERNS:
+        cleaned = re.sub(
+            pattern,
+            " ",
+            cleaned,
+            flags=re.IGNORECASE
+        )
+
+    cleaned = cleaned.replace("\n", " ")
+    cleaned = re.sub(r"\s+", " ", cleaned)
+    cleaned = cleaned.replace(" .", ".")
+    cleaned = cleaned.replace(" ,", ",")
+    cleaned = cleaned.strip()
+
+    return cleaned
+
+
+def clean_sentence_text(sentence: str) -> str:
+    """
+    Cleans one evidence sentence.
+    """
+
+    if not sentence:
+        return ""
+
+    cleaned = sentence
+
+    cleaned = re.sub(r"^Q\d+\s*:\s*", "", cleaned, flags=re.IGNORECASE)
+    cleaned = re.sub(r"^Ideal Answer\s*", "", cleaned, flags=re.IGNORECASE)
+    cleaned = re.sub(r"\s+", " ", cleaned)
+
+    cleaned = cleaned.replace(" .", ".")
+    cleaned = cleaned.replace(" ,", ",")
+    cleaned = cleaned.strip()
+
+    return cleaned
+
+
+def clean_retrieved_results(results: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    cleaned_results = []
+
+    for result in results:
+        result = dict(result)
+        result["raw_content"] = result.get("content", "")
+        result["content"] = clean_chunk_text(result.get("content", ""))
+        cleaned_results.append(result)
+
+    return cleaned_results

app/generation/evidence_extractor.py

@@ -0,0 +1,206 @@
+import re
+from typing import List, Dict, Any
+
+from app.generation.context_cleaner import clean_sentence_text
+
+
+STOPWORDS = {
+    "what", "is", "are", "the", "a", "an", "of", "to", "and", "or", "in",
+    "for", "with", "on", "by", "from", "this", "that", "it", "does",
+    "do", "why", "how", "explain", "define", "meaning"
+}
+
+
+def split_into_sentences(text: str) -> List[str]:
+    if not text:
+        return []
+
+    sentence_candidates = re.split(r"(?<=[.!?])\s+", text)
+    sentences = []
+
+    for sentence in sentence_candidates:
+        sentence = clean_sentence_text(sentence)
+
+        if len(sentence) < 25:
+            continue
+
+        if is_noise_sentence(sentence):
+            continue
+
+        sentences.append(sentence)
+
+    return sentences
+
+
+def is_noise_sentence(sentence: str) -> bool:
+    sentence_lower = sentence.lower().strip()
+
+    noise_starts = [
+        "chapter ",
+        "page ",
+        "this chapter prepares",
+        "practice saying",
+        "component what it does",
+    ]
+
+    for start in noise_starts:
+        if sentence_lower.startswith(start):
+            return True
+
+    return False
+
+
+def extract_query_terms(query: str) -> List[str]:
+    words = re.findall(r"[a-zA-Z0-9_]+", query.lower())
+
+    terms = [
+        word for word in words
+        if word not in STOPWORDS and len(word) > 1
+    ]
+
+    return terms
+
+
+def score_sentence(sentence: str, query_terms: List[str]) -> float:
+    sentence_lower = sentence.lower()
+    score = 0.0
+
+    for term in query_terms:
+        if term in sentence_lower:
+            score += 2.0
+
+    important_markers = [
+        "stands for",
+        "means",
+        "refers to",
+        "retrieval-augmented generation",
+        "retrieval augmented generation",
+        "adds a retrieval step",
+        "adding a retrieval step",
+        "before generation",
+        "before generating",
+        "search your document corpus",
+        "search a document corpus",
+        "provide the relevant passages",
+        "relevant passages as context",
+        "frozen knowledge",
+        "reduces hallucination",
+        "grounds the answer",
+        "private or recent data"
+    ]
+
+    for marker in important_markers:
+        if marker in sentence_lower:
+            score += 1.5
+
+    if 60 <= len(sentence) <= 350:
+        score += 0.5
+
+    return score
+
+
+def normalize_for_dedup(text: str) -> str:
+    text = text.lower()
+    text = re.sub(r"[^a-z0-9\s]", " ", text)
+    text = re.sub(r"\s+", " ", text).strip()
+    return text
+
+
+def token_set(text: str) -> set:
+    return set(normalize_for_dedup(text).split())
+
+
+def is_similar_to_existing(sentence: str, existing_sentences: List[str]) -> bool:
+    current_tokens = token_set(sentence)
+
+    if not current_tokens:
+        return True
+
+    for existing in existing_sentences:
+        existing_tokens = token_set(existing)
+
+        if not existing_tokens:
+            continue
+
+        overlap = len(current_tokens.intersection(existing_tokens))
+        union = len(current_tokens.union(existing_tokens))
+
+        if union == 0:
+            continue
+
+        similarity = overlap / union
+
+        if similarity >= 0.72:
+            return True
+
+    return False
+
+
+def extract_evidence_sentences(
+    query: str,
+    results: List[Dict[str, Any]],
+    max_evidence: int = 8
+) -> List[Dict[str, Any]]:
+
+    query_terms = extract_query_terms(query)
+    evidence_items = []
+
+    for result in results:
+        content = result.get("content", "")
+        sentences = split_into_sentences(content)
+
+        for sentence in sentences:
+            score = score_sentence(sentence, query_terms)
+
+            if score <= 0:
+                continue
+
+            evidence_items.append(
+                {
+                    "sentence": sentence,
+                    "score": score,
+                    "source_id": result.get("source_id"),
+                    "citation": result.get("citation"),
+                    "chunk_id": result.get("chunk_id"),
+                    "document_id": result.get("document_id"),
+                    "source_file_name": result.get("source_file_name"),
+                    "page_number": result.get("page_number")
+                }
+            )
+
+    evidence_items.sort(
+        key=lambda item: item["score"],
+        reverse=True
+    )
+
+    deduplicated = []
+    existing_sentences = []
+
+    for item in evidence_items:
+        sentence = item["sentence"]
+
+        if is_similar_to_existing(sentence, existing_sentences):
+            continue
+
+        deduplicated.append(item)
+        existing_sentences.append(sentence)
+
+        if len(deduplicated) >= max_evidence:
+            break
+
+    return deduplicated
+
+
+def build_evidence_context(evidence_items: List[Dict[str, Any]]) -> str:
+    context_lines = []
+
+    for item in evidence_items:
+        source_id = item.get("source_id", "S?")
+        citation = item.get("citation", "")
+        sentence = item.get("sentence", "")
+
+        context_lines.append(
+            f"{source_id}: {sentence} {citation}"
+        )
+
+    return "\n".join(context_lines)

app/generation/llm_service.py

@@ -0,0 +1,52 @@
+from typing import Dict, Any
+
+from app.core.config import settings
+from app.generation.provider_factory import get_llm_provider
+
+
+def generate_with_local_llm(prompt: str) -> str:
+    """
+    Backward-compatible function name.
+
+    Earlier answer_service.py calls generate_with_local_llm().
+    Now this routes to the configured provider:
+    - local
+    - huggingface
+    - disabled
+    """
+
+    provider = get_llm_provider()
+    return provider.generate(prompt)
+
+
+def generate_with_configured_llm(prompt: str) -> str:
+    provider = get_llm_provider()
+    return provider.generate(prompt)
+
+
+def get_llm_status() -> Dict[str, Any]:
+    provider = get_llm_provider()
+    provider_status = provider.status()
+
+    return {
+        "active_provider": settings.LLM_PROVIDER,
+        "provider_status": provider_status,
+        "available_providers": [
+            "local",
+            "huggingface",
+            "disabled"
+        ],
+        "future_providers": [
+            "aws_bedrock",
+            "openai"
+        ],
+        "fallback_behavior": (
+            "If the provider returns a weak or empty answer, "
+            "answer_service uses evidence-based fallback."
+        )
+    }
+
+
+def get_loaded_llm_info() -> Dict[str, Any]:
+    provider = get_llm_provider()
+    return provider.load_test()

app/generation/prompt_builder.py

@@ -0,0 +1,37 @@
+from app.generation.question_classifier import get_answer_instruction
+
+
+def build_grounded_prompt(
+    query: str,
+    evidence_context: str,
+    question_type: str
+) -> str:
+    """
+    Builds a compact prompt.
+
+    Small local models perform better with short, direct prompts.
+    """
+
+    instruction = get_answer_instruction(question_type)
+
+    return f"""
+Answer the question using only the evidence.
+
+Question type: {question_type}
+
+Instruction: {instruction}
+
+Rules:
+- Do not use outside knowledge.
+- Do not mention missing information unless evidence is missing.
+- Use citations like [S1] and [S2].
+- Give a clear final answer, not notes.
+
+Question:
+{query}
+
+Evidence:
+{evidence_context}
+
+Final answer:
+""".strip()

app/generation/provider_factory.py

@@ -0,0 +1,29 @@
+from app.core.config import settings
+from app.generation.providers.base_provider import BaseLLMProvider
+from app.generation.providers.local_provider import LocalLLMProvider
+from app.generation.providers.huggingface_provider import HuggingFaceLLMProvider
+from app.generation.providers.disabled_provider import DisabledLLMProvider
+
+
+def get_llm_provider() -> BaseLLMProvider:
+    """
+    Selects the active LLM provider using settings.LLM_PROVIDER.
+
+    Supported:
+    - local
+    - huggingface
+    - disabled
+    """
+
+    provider_name = settings.LLM_PROVIDER.lower().strip()
+
+    if provider_name == "local":
+        return LocalLLMProvider()
+
+    if provider_name in ["hf", "huggingface", "hugging_face"]:
+        return HuggingFaceLLMProvider()
+
+    if provider_name in ["none", "off", "disabled"]:
+        return DisabledLLMProvider()
+
+    return DisabledLLMProvider()

app/generation/providers/base_provider.py

@@ -0,0 +1,27 @@
+from abc import ABC, abstractmethod
+from typing import Dict, Any
+
+
+class BaseLLMProvider(ABC):
+    """
+    Base interface for all LLM providers.
+
+    Every provider must implement:
+    - generate()
+    - status()
+    - load_test()
+    """
+
+    provider_name: str = "base"
+
+    @abstractmethod
+    def generate(self, prompt: str) -> str:
+        pass
+
+    @abstractmethod
+    def status(self) -> Dict[str, Any]:
+        pass
+
+    @abstractmethod
+    def load_test(self) -> Dict[str, Any]:
+        pass

app/generation/providers/disabled_provider.py

@@ -0,0 +1,24 @@
+from typing import Dict, Any
+
+from app.generation.providers.base_provider import BaseLLMProvider
+
+
+class DisabledLLMProvider(BaseLLMProvider):
+    provider_name = "disabled"
+
+    def generate(self, prompt: str) -> str:
+        return ""
+
+    def status(self) -> Dict[str, Any]:
+        return {
+            "provider": self.provider_name,
+            "enabled": False,
+            "message": "LLM provider is disabled. Evidence-based fallback will be used."
+        }
+
+    def load_test(self) -> Dict[str, Any]:
+        return {
+            "loaded": False,
+            "provider": self.provider_name,
+            "message": "LLM provider is disabled."
+        }

app/generation/providers/huggingface_provider.py

@@ -0,0 +1,133 @@
+from typing import Dict, Any
+import requests
+import re
+
+from app.core.config import settings
+from app.generation.providers.base_provider import BaseLLMProvider
+
+
+class HuggingFaceLLMProvider(BaseLLMProvider):
+    provider_name = "huggingface"
+
+    def generate(self, prompt: str) -> str:
+        if not settings.HF_API_TOKEN:
+            return ""
+
+        try:
+            url = get_hf_inference_url()
+
+            headers = {
+                "Authorization": f"Bearer {settings.HF_API_TOKEN}",
+                "Content-Type": "application/json"
+            }
+
+            payload = {
+                "inputs": prompt,
+                "parameters": {
+                    "max_new_tokens": settings.MAX_GENERATION_TOKENS,
+                    "do_sample": False,
+                    "return_full_text": False
+                }
+            }
+
+            response = requests.post(
+                url=url,
+                headers=headers,
+                json=payload,
+                timeout=settings.HF_TIMEOUT_SECONDS
+            )
+
+            if response.status_code != 200:
+                return ""
+
+            data = response.json()
+            answer = parse_huggingface_response(data)
+
+            return clean_hosted_output(answer)
+
+        except Exception:
+            return ""
+
+    def status(self) -> Dict[str, Any]:
+        return {
+            "provider": self.provider_name,
+            "enabled": bool(settings.HF_API_TOKEN),
+            "model_name": settings.HF_INFERENCE_MODEL,
+            "custom_url_set": bool(settings.HF_INFERENCE_URL),
+            "timeout_seconds": settings.HF_TIMEOUT_SECONDS,
+            "token_present": bool(settings.HF_API_TOKEN)
+        }
+
+    def load_test(self) -> Dict[str, Any]:
+        if not settings.HF_API_TOKEN:
+            return {
+                "loaded": False,
+                "provider": self.provider_name,
+                "message": "HF_API_TOKEN is missing."
+            }
+
+        try:
+            test_prompt = "Answer briefly: What is RAG?"
+
+            answer = self.generate(test_prompt)
+
+            return {
+                "loaded": bool(answer),
+                "provider": self.provider_name,
+                "model_name": settings.HF_INFERENCE_MODEL,
+                "answer_preview": answer[:200],
+                "message": "Hugging Face provider call completed."
+            }
+
+        except Exception as error:
+            return {
+                "loaded": False,
+                "provider": self.provider_name,
+                "model_name": settings.HF_INFERENCE_MODEL,
+                "error": str(error)
+            }
+
+
+def get_hf_inference_url() -> str:
+    if settings.HF_INFERENCE_URL:
+        return settings.HF_INFERENCE_URL
+
+    return f"https://api-inference.huggingface.co/models/{settings.HF_INFERENCE_MODEL}"
+
+
+def parse_huggingface_response(data) -> str:
+    if isinstance(data, list) and data:
+        first_item = data[0]
+
+        if isinstance(first_item, dict):
+            if "generated_text" in first_item:
+                return str(first_item["generated_text"])
+
+            if "summary_text" in first_item:
+                return str(first_item["summary_text"])
+
+    if isinstance(data, dict):
+        if "generated_text" in data:
+            return str(data["generated_text"])
+
+        if "summary_text" in data:
+            return str(data["summary_text"])
+
+        if "error" in data:
+            return ""
+
+    return ""
+
+
+def clean_hosted_output(answer: str) -> str:
+    if not answer:
+        return ""
+
+    cleaned = answer.strip()
+
+    cleaned = re.sub(r"\s+", " ", cleaned)
+    cleaned = cleaned.replace(" .", ".")
+    cleaned = cleaned.replace(" ,", ",")
+    cleaned = cleaned.strip()
+
+    return cleaned

app/generation/providers/local_provider.py

@@ -0,0 +1,185 @@
+from functools import lru_cache
+from typing import Dict, Any
+import re
+
+import torch
+from transformers import (
+    AutoTokenizer,
+    AutoConfig,
+    AutoModelForSeq2SeqLM,
+    AutoModelForCausalLM
+)
+
+from app.core.config import settings
+from app.generation.providers.base_provider import BaseLLMProvider
+
+
+class LocalLLMProvider(BaseLLMProvider):
+    provider_name = "local"
+
+    def generate(self, prompt: str) -> str:
+        if not settings.ENABLE_LOCAL_LLM:
+            return ""
+
+        try:
+            llm_bundle = get_local_llm()
+
+            tokenizer = llm_bundle["tokenizer"]
+            model = llm_bundle["model"]
+            model_type = llm_bundle["model_type"]
+
+            if model_type == "seq2seq":
+                answer = generate_seq2seq_answer(
+                    tokenizer=tokenizer,
+                    model=model,
+                    prompt=prompt
+                )
+            else:
+                answer = generate_causal_answer(
+                    tokenizer=tokenizer,
+                    model=model,
+                    prompt=prompt
+                )
+
+            return clean_llm_output(answer)
+
+        except Exception:
+            return ""
+
+    def status(self) -> Dict[str, Any]:
+        return {
+            "provider": self.provider_name,
+            "enabled": settings.ENABLE_LOCAL_LLM,
+            "model_name": settings.LOCAL_LLM_MODEL_NAME,
+            "device": settings.LOCAL_LLM_DEVICE,
+            "max_generation_tokens": settings.MAX_GENERATION_TOKENS,
+            "max_input_tokens": settings.LOCAL_LLM_MAX_INPUT_TOKENS,
+            "min_answer_words": settings.MIN_LLM_ANSWER_WORDS
+        }
+
+    def load_test(self) -> Dict[str, Any]:
+        try:
+            llm_bundle = get_local_llm()
+
+            return {
+                "loaded": True,
+                "provider": self.provider_name,
+                "model_name": llm_bundle["model_name"],
+                "model_type": llm_bundle["model_type"],
+                "enabled": settings.ENABLE_LOCAL_LLM
+            }
+
+        except Exception as error:
+            return {
+                "loaded": False,
+                "provider": self.provider_name,
+                "model_name": settings.LOCAL_LLM_MODEL_NAME,
+                "error": str(error)
+            }
+
+
+@lru_cache(maxsize=1)
+def get_local_llm():
+    tokenizer = AutoTokenizer.from_pretrained(settings.LOCAL_LLM_MODEL_NAME)
+    config = AutoConfig.from_pretrained(settings.LOCAL_LLM_MODEL_NAME)
+
+    if getattr(config, "is_encoder_decoder", False):
+        model = AutoModelForSeq2SeqLM.from_pretrained(
+            settings.LOCAL_LLM_MODEL_NAME
+        )
+        model_type = "seq2seq"
+    else:
+        model = AutoModelForCausalLM.from_pretrained(
+            settings.LOCAL_LLM_MODEL_NAME
+        )
+        model_type = "causal"
+
+    model.eval()
+
+    return {
+        "tokenizer": tokenizer,
+        "model": model,
+        "model_type": model_type,
+        "model_name": settings.LOCAL_LLM_MODEL_NAME
+    }
+
+
+def generate_seq2seq_answer(tokenizer, model, prompt: str) -> str:
+    inputs = tokenizer(
+        prompt,
+        return_tensors="pt",
+        truncation=True,
+        max_length=settings.LOCAL_LLM_MAX_INPUT_TOKENS
+    )
+
+    with torch.no_grad():
+        output_ids = model.generate(
+            **inputs,
+            max_new_tokens=settings.MAX_GENERATION_TOKENS,
+            do_sample=False,
+            num_beams=4,
+            early_stopping=True
+        )
+
+    answer = tokenizer.decode(
+        output_ids[0],
+        skip_special_tokens=True
+    )
+
+    return answer
+
+
+def generate_causal_answer(tokenizer, model, prompt: str) -> str:
+    if tokenizer.pad_token is None:
+        tokenizer.pad_token = tokenizer.eos_token
+
+    inputs = tokenizer(
+        prompt,
+        return_tensors="pt",
+        truncation=True,
+        max_length=settings.LOCAL_LLM_MAX_INPUT_TOKENS
+    )
+
+    input_length = inputs["input_ids"].shape[-1]
+
+    with torch.no_grad():
+        output_ids = model.generate(
+            **inputs,
+            max_new_tokens=settings.MAX_GENERATION_TOKENS,
+            do_sample=False,
+            pad_token_id=tokenizer.eos_token_id
+        )
+
+    generated_ids = output_ids[0][input_length:]
+
+    answer = tokenizer.decode(
+        generated_ids,
+        skip_special_tokens=True
+    )
+
+    return answer
+
+
+def clean_llm_output(answer: str) -> str:
+    if not answer:
+        return ""
+
+    cleaned = answer.strip()
+
+    unwanted_prefixes = [
+        "final answer:",
+        "answer:",
+        "the answer is:",
+        "output:"
+    ]
+
+    for prefix in unwanted_prefixes:
+        if cleaned.lower().startswith(prefix):
+            cleaned = cleaned[len(prefix):].strip()
+
+    cleaned = re.sub(r"\s+", " ", cleaned)
+    cleaned = cleaned.replace(" .", ".")
+    cleaned = cleaned.replace(" ,", ",")
+    cleaned = cleaned.strip()
+
+    return cleaned

app/generation/question_classifier.py

@@ -0,0 +1,69 @@
+from typing import Literal
+
+
+QuestionType = Literal[
+    "definition",
+    "summary",
+    "comparison",
+    "steps",
+    "reason",
+    "general"
+]
+
+
+def classify_question(query: str) -> QuestionType:
+    query_lower = query.lower().strip()
+
+    if query_lower.startswith("what is") or query_lower.startswith("what are"):
+        return "definition"
+
+    if query_lower.startswith("define") or "meaning of" in query_lower:
+        return "definition"
+
+    if "summarize" in query_lower or "summary" in query_lower:
+        return "summary"
+
+    if "compare" in query_lower or "difference between" in query_lower or "vs" in query_lower:
+        return "comparison"
+
+    if query_lower.startswith("how to") or "steps" in query_lower or "step by step" in query_lower:
+        return "steps"
+
+    if query_lower.startswith("why") or "reason" in query_lower:
+        return "reason"
+
+    return "general"
+
+
+def get_answer_instruction(question_type: QuestionType) -> str:
+    if question_type == "definition":
+        return (
+            "Give a clear definition first. Then explain why it matters. "
+            "Keep the answer short and cite the evidence."
+        )
+
+    if question_type == "summary":
+        return (
+            "Give a concise structured summary using the most important points. "
+            "Avoid unnecessary details and cite sources."
+        )
+
+    if question_type == "comparison":
+        return (
+            "Compare the concepts clearly. Mention similarities, differences, "
+            "and practical implications. Cite sources."
+        )
+
+    if question_type == "steps":
+        return (
+            "Explain the answer as clear steps. Keep each step simple. Cite sources."
+        )
+
+    if question_type == "reason":
+        return (
+            "Explain the reason logically using evidence from the sources. Cite sources."
+        )
+
+    return (
+        "Answer clearly using only the retrieved sources. Cite the evidence."
+    )

app/ingestion/base_parser.py

@@ -0,0 +1,15 @@
+from abc import ABC, abstractmethod
+from typing import List
+from app.schemas.rich_content_block import RichContentBlock
+
+
+class BaseParser(ABC):
+
+    @abstractmethod
+    def parse(
+        self,
+        file_path: str,
+        document_id: str,
+        source_file_name: str
+    ) -> List[RichContentBlock]:
+        pass

app/ingestion/csv_excel_parser.py

@@ -0,0 +1,107 @@
+from pathlib import Path
+from typing import List, Dict, Any
+import pandas as pd
+
+from app.core.config import settings
+from app.ingestion.base_parser import BaseParser
+from app.schemas.rich_content_block import RichContentBlock
+
+
+class CsvExcelParser(BaseParser):
+
+    def parse(
+        self,
+        file_path: str,
+        document_id: str,
+        source_file_name: str
+    ) -> List[RichContentBlock]:
+
+        extension = Path(source_file_name).suffix.lower()
+
+        if extension == ".csv":
+            sheets = {"csv": read_csv_safely(file_path)}
+        elif extension in [".xlsx", ".xls"]:
+            sheets = pd.read_excel(file_path, sheet_name=None)
+        else:
+            raise ValueError(f"Unsupported table file extension: {extension}")
+
+        blocks = []
+        block_counter = 1
+
+        for sheet_name, dataframe in sheets.items():
+            dataframe = clean_dataframe(dataframe)
+
+            if dataframe.empty:
+                continue
+
+            for start_row in range(0, len(dataframe), settings.MAX_ROWS_PER_TABLE_BLOCK):
+                end_row = min(start_row + settings.MAX_ROWS_PER_TABLE_BLOCK, len(dataframe))
+                batch_df = dataframe.iloc[start_row:end_row]
+
+                table_json = batch_df.to_dict(orient="records")
+                markdown_table = dataframe_to_markdown(batch_df)
+
+                blocks.append(
+                    RichContentBlock(
+                        block_id=f"{document_id}_table_block_{block_counter}",
+                        document_id=document_id,
+                        content_type="table",
+                        content=markdown_table,
+                        page_number=None,
+                        section_title=f"Sheet: {sheet_name}",
+                        source_file_name=source_file_name,
+                        metadata={
+                            "parser": "CsvExcelParser",
+                            "original_format": extension,
+                            "sheet_name": sheet_name,
+                            "table_json": table_json,
+                            "columns": list(batch_df.columns),
+                            "row_start": start_row + 1,
+                            "row_end": end_row,
+                            "total_rows_in_sheet": len(dataframe),
+                            "max_rows_per_table_block": settings.MAX_ROWS_PER_TABLE_BLOCK
+                        }
+                    )
+                )
+                block_counter += 1
+
+        return blocks
+
+
+def read_csv_safely(file_path: str) -> pd.DataFrame:
+    try:
+        return pd.read_csv(file_path)
+    except UnicodeDecodeError:
+        return pd.read_csv(file_path, encoding="latin1")
+
+
+def clean_dataframe(dataframe: pd.DataFrame) -> pd.DataFrame:
+    dataframe = dataframe.copy()
+    dataframe = dataframe.dropna(how="all")
+    dataframe = dataframe.dropna(axis=1, how="all")
+    dataframe.columns = [
+        str(column).strip() if str(column).strip() else f"column_{index + 1}"
+        for index, column in enumerate(dataframe.columns)
+    ]
+    dataframe = dataframe.fillna("")
+
+    for column in dataframe.columns:
+        dataframe[column] = dataframe[column].astype(str)
+
+    return dataframe
+
+
+def dataframe_to_markdown(dataframe: pd.DataFrame) -> str:
+    columns = [str(column) for column in dataframe.columns]
+    lines = []
+    lines.append("| " + " | ".join(columns) + " |")
+    lines.append("| " + " | ".join(["---"] * len(columns)) + " |")
+
+    for _, row in dataframe.iterrows():
+        values = [
+            str(row[column]).replace("|", "\\|").replace("\n", " ").strip()
+            for column in columns
+        ]
+        lines.append("| " + " | ".join(values) + " |")
+
+    return "\n".join(lines)

app/ingestion/docx_parser.py

@@ -0,0 +1,120 @@
+from typing import List
+from docx import Document
+from docx.text.paragraph import Paragraph
+from docx.table import Table
+from docx.oxml.text.paragraph import CT_P
+from docx.oxml.table import CT_Tbl
+
+from app.ingestion.base_parser import BaseParser
+from app.schemas.rich_content_block import RichContentBlock
+
+
+class DocxParser(BaseParser):
+
+    def parse(
+        self,
+        file_path: str,
+        document_id: str,
+        source_file_name: str
+    ) -> List[RichContentBlock]:
+
+        doc = Document(file_path)
+        blocks = []
+        block_counter = 1
+        current_section_title = None
+
+        for element in doc.element.body:
+            if isinstance(element, CT_P):
+                paragraph = Paragraph(element, doc)
+                text = paragraph.text.strip()
+
+                if not text:
+                    continue
+
+                style_name = paragraph.style.name if paragraph.style else ""
+
+                if style_name.startswith("Heading"):
+                    current_section_title = text
+
+                blocks.append(
+                    RichContentBlock(
+                        block_id=f"{document_id}_docx_block_{block_counter}",
+                        document_id=document_id,
+                        content_type="text",
+                        content=text,
+                        page_number=None,
+                        section_title=current_section_title,
+                        source_file_name=source_file_name,
+                        metadata={
+                            "parser": "DocxParser",
+                            "original_format": "docx",
+                            "docx_element_type": "paragraph",
+                            "style": style_name
+                        }
+                    )
+                )
+                block_counter += 1
+
+            elif isinstance(element, CT_Tbl):
+                table = Table(element, doc)
+                table_data = extract_table_as_rows(table)
+                markdown_table = rows_to_markdown(table_data)
+
+                if not markdown_table:
+                    continue
+
+                blocks.append(
+                    RichContentBlock(
+                        block_id=f"{document_id}_docx_block_{block_counter}",
+                        document_id=document_id,
+                        content_type="table",
+                        content=markdown_table,
+                        page_number=None,
+                        section_title=current_section_title,
+                        source_file_name=source_file_name,
+                        metadata={
+                            "parser": "DocxParser",
+                            "original_format": "docx",
+                            "docx_element_type": "table",
+                            "table_json": table_data
+                        }
+                    )
+                )
+                block_counter += 1
+
+        return blocks
+
+
+def extract_table_as_rows(table: Table):
+    rows = []
+
+    for row in table.rows:
+        rows.append([cell.text.strip() for cell in row.cells])
+
+    return rows
+
+
+def rows_to_markdown(rows):
+    if not rows:
+        return ""
+
+    header = rows[0]
+    body = rows[1:]
+
+    lines = []
+    lines.append("| " + " | ".join(header) + " |")
+    lines.append("| " + " | ".join(["---"] * len(header)) + " |")
+
+    for row in body:
+        row = normalize_row(row, len(header))
+        lines.append("| " + " | ".join(row) + " |")
+
+    return "\n".join(lines)
+
+
+def normalize_row(row, expected_len):
+    if len(row) < expected_len:
+        row = row + [""] * (expected_len - len(row))
+    if len(row) > expected_len:
+        row = row[:expected_len]
+    return row

app/ingestion/file_detector.py

@@ -0,0 +1,25 @@
+from pathlib import Path
+
+
+SUPPORTED_FILE_TYPES = {
+    ".txt": "txt",
+    ".md": "markdown",
+    ".markdown": "markdown",
+    ".pdf": "pdf",
+    ".docx": "docx",
+    ".csv": "csv",
+    ".xlsx": "excel",
+    ".xls": "excel",
+    ".png": "image",
+    ".jpg": "image",
+    ".jpeg": "image",
+    ".webp": "image",
+    ".html": "html",
+    ".htm": "html",
+    ".tex": "latex",
+}
+
+
+def detect_file_type(filename: str) -> str:
+    extension = Path(filename).suffix.lower()
+    return SUPPORTED_FILE_TYPES.get(extension, "unsupported")

app/ingestion/html_parser.py

@@ -0,0 +1,104 @@
+from typing import List
+from bs4 import BeautifulSoup
+
+from app.ingestion.base_parser import BaseParser
+from app.schemas.rich_content_block import RichContentBlock
+
+
+class HtmlParser(BaseParser):
+
+    def parse(
+        self,
+        file_path: str,
+        document_id: str,
+        source_file_name: str
+    ) -> List[RichContentBlock]:
+
+        with open(file_path, "r", encoding="utf-8", errors="ignore") as f:
+            html = f.read()
+
+        soup = BeautifulSoup(html, "lxml")
+
+        for tag in soup(["script", "style", "noscript", "svg"]):
+            tag.decompose()
+
+        blocks = []
+        block_counter = 1
+        current_section_title = None
+
+        for element in soup.find_all(["h1", "h2", "h3", "h4", "p", "li", "blockquote", "pre", "code", "table"]):
+            tag_name = element.name.lower()
+
+            if tag_name in ["h1", "h2", "h3", "h4"]:
+                text = clean_text(element.get_text(" ", strip=True))
+                current_section_title = text
+                content_type = "text"
+
+            elif tag_name == "table":
+                rows = []
+                for tr in element.find_all("tr"):
+                    cells = [
+                        clean_text(cell.get_text(" ", strip=True))
+                        for cell in tr.find_all(["th", "td"])
+                    ]
+                    if cells:
+                        rows.append(cells)
+
+                text = rows_to_markdown(rows)
+                content_type = "table"
+
+            elif tag_name in ["pre", "code"]:
+                text = element.get_text("\n", strip=True)
+                content_type = "code"
+
+            else:
+                text = clean_text(element.get_text(" ", strip=True))
+                content_type = "text"
+
+            if not text:
+                continue
+
+            blocks.append(
+                RichContentBlock(
+                    block_id=f"{document_id}_html_block_{block_counter}",
+                    document_id=document_id,
+                    content_type=content_type,
+                    content=text,
+                    page_number=None,
+                    section_title=current_section_title,
+                    source_file_name=source_file_name,
+                    metadata={
+                        "parser": "HtmlParser",
+                        "original_format": "html",
+                        "html_tag": tag_name
+                    }
+                )
+            )
+            block_counter += 1
+
+        return blocks
+
+
+def clean_text(text: str) -> str:
+    return " ".join(text.split())
+
+
+def rows_to_markdown(rows):
+    if not rows:
+        return ""
+
+    header = rows[0]
+    body = rows[1:]
+
+    lines = []
+    lines.append("| " + " | ".join(header) + " |")
+    lines.append("| " + " | ".join(["---"] * len(header)) + " |")
+
+    for row in body:
+        if len(row) < len(header):
+            row = row + [""] * (len(header) - len(row))
+        if len(row) > len(header):
+            row = row[:len(header)]
+        lines.append("| " + " | ".join(row) + " |")
+
+    return "\n".join(lines)

app/ingestion/image_parser.py

@@ -0,0 +1,69 @@
+import shutil
+from pathlib import Path
+from typing import List
+
+from PIL import Image
+
+from app.core.config import settings
+from app.ingestion.base_parser import BaseParser
+from app.schemas.rich_content_block import RichContentBlock
+
+
+class ImageParser(BaseParser):
+
+    def parse(
+        self,
+        file_path: str,
+        document_id: str,
+        source_file_name: str
+    ) -> List[RichContentBlock]:
+
+        source_path = Path(file_path)
+
+        image_assets_dir = settings.PROCESSED_DIR / document_id / "assets" / "images"
+        image_assets_dir.mkdir(parents=True, exist_ok=True)
+
+        safe_image_name = Path(source_file_name).name
+        stored_image_path = image_assets_dir / safe_image_name
+
+        shutil.copy2(source_path, stored_image_path)
+
+        image_metadata = extract_image_metadata(stored_image_path)
+
+        return [
+            RichContentBlock(
+                block_id=f"{document_id}_image_block_1",
+                document_id=document_id,
+                content_type="image",
+                content=f"Image file uploaded: {source_file_name}. Caption not generated yet.",
+                page_number=None,
+                section_title=None,
+                source_file_name=source_file_name,
+                metadata={
+                    "parser": "ImageParser",
+                    "original_format": image_metadata["image_format"],
+                    "image_path": str(stored_image_path),
+                    "image_file_name": safe_image_name,
+                    "width": image_metadata["width"],
+                    "height": image_metadata["height"],
+                    "mode": image_metadata["mode"],
+                    "file_size_bytes": image_metadata["file_size_bytes"],
+                    "caption_status": "not_generated"
+                }
+            )
+        ]
+
+
+def extract_image_metadata(image_path: Path) -> dict:
+    with Image.open(image_path) as image:
+        width, height = image.size
+        image_format = image.format
+        mode = image.mode
+
+    return {
+        "width": width,
+        "height": height,
+        "image_format": image_format,
+        "mode": mode,
+        "file_size_bytes": image_path.stat().st_size
+    }

app/ingestion/ingestion_service.py

@@ -0,0 +1,247 @@
+import uuid
+import hashlib
+from pathlib import Path
+from fastapi import UploadFile
+
+from app.core.config import settings
+from app.ingestion.file_detector import detect_file_type
+from app.ingestion.parser_registry import parser_registry
+from app.chunking.chunking_service import chunk_blocks
+from app.storage.processed_storage import save_processed_document
+from app.storage.status_storage import (
+    create_document_status,
+    update_document_status
+)
+from app.storage.document_index import (
+    find_duplicate_by_hash,
+    register_document_hash
+)
+
+
+async def save_uploaded_file(file: UploadFile, document_id: str) -> tuple[str, str, int]:
+    settings.UPLOAD_DIR.mkdir(parents=True, exist_ok=True)
+
+    safe_filename = Path(file.filename).name
+    saved_filename = f"{document_id}_{safe_filename}"
+    file_path = settings.UPLOAD_DIR / saved_filename
+
+    content = await file.read()
+
+    upload_size_bytes = len(content)
+    max_upload_size_bytes = settings.MAX_UPLOAD_SIZE_MB * 1024 * 1024
+
+    if upload_size_bytes > max_upload_size_bytes:
+        raise ValueError(
+            f"File is too large. Maximum allowed size is "
+            f"{settings.MAX_UPLOAD_SIZE_MB} MB, but uploaded file is "
+            f"{round(upload_size_bytes / (1024 * 1024), 2)} MB."
+        )
+
+    file_hash = hashlib.sha256(content).hexdigest()
+
+    with open(file_path, "wb") as f:
+        f.write(content)
+
+    return str(file_path), file_hash, upload_size_bytes
+
+
+async def process_uploaded_file(file: UploadFile):
+    document_id = str(uuid.uuid4())
+
+    try:
+        file_path, file_hash, upload_size_bytes = await save_uploaded_file(
+            file=file,
+            document_id=document_id
+        )
+
+        duplicate_document = find_duplicate_by_hash(file_hash)
+
+        if duplicate_document is not None:
+            try:
+                Path(file_path).unlink(missing_ok=True)
+            except Exception:
+                pass
+
+            return {
+                "status": "duplicate",
+                "message": "This exact file was already uploaded and processed.",
+                "uploaded_file_name": file.filename,
+                "existing_document": duplicate_document,
+                "duplicate_detection": {
+                    "method": "sha256_file_hash",
+                    "file_hash": file_hash
+                }
+            }
+
+        create_document_status(
+            document_id=document_id,
+            source_file_name=file.filename
+        )
+
+        file_type = detect_file_type(file.filename)
+
+        update_document_status(
+            document_id=document_id,
+            status="processing",
+            current_stage="file_type_detected",
+            file_type=file_type,
+            message=f"Detected file type: {file_type}",
+            metadata={
+                "uploaded_file_path": file_path,
+                "file_hash": file_hash,
+                "upload_size_bytes": upload_size_bytes,
+                "upload_size_mb": round(upload_size_bytes / (1024 * 1024), 2),
+                "max_upload_size_mb": settings.MAX_UPLOAD_SIZE_MB
+            }
+        )
+
+        parser = parser_registry.get_parser(file_type)
+
+        if parser is None:
+            update_document_status(
+                document_id=document_id,
+                status="failed",
+                current_stage="parser_not_available",
+                file_type=file_type,
+                message="Parser for this file type is not implemented yet.",
+                error_message="No parser registered for this file type."
+            )
+
+            return {
+                "status": "failed",
+                "document_id": document_id,
+                "file_name": file.filename,
+                "file_type": file_type,
+                "message": "Parser for this file type is not implemented yet."
+            }
+
+        update_document_status(
+            document_id=document_id,
+            status="processing",
+            current_stage="parsing",
+            file_type=file_type,
+            message="Parsing document into RichContentBlocks."
+        )
+
+        blocks = parser.parse(
+            file_path=file_path,
+            document_id=document_id,
+            source_file_name=file.filename
+        )
+
+        update_document_status(
+            document_id=document_id,
+            status="processing",
+            current_stage="chunking",
+            file_type=file_type,
+            message="Chunking RichContentBlocks.",
+            metadata={
+                "blocks_created": len(blocks)
+            }
+        )
+
+        chunks = chunk_blocks(blocks)
+
+        update_document_status(
+            document_id=document_id,
+            status="processing",
+            current_stage="saving",
+            file_type=file_type,
+            message="Saving processed document.",
+            metadata={
+                "chunks_created": len(chunks)
+            }
+        )
+
+        processed_metadata = save_processed_document(
+            document_id=document_id,
+            source_file_name=file.filename,
+            file_type=file_type,
+            file_hash=file_hash,
+            blocks=blocks,
+            chunks=chunks
+        )
+
+        register_document_hash(
+            document_id=document_id,
+            source_file_name=file.filename,
+            file_type=file_type,
+            file_hash=file_hash
+        )
+
+        update_document_status(
+            document_id=document_id,
+            status="processed",
+            current_stage="processed",
+            file_type=file_type,
+            message="Document processed successfully.",
+            metadata={
+                "file_hash": file_hash,
+                "processed_files": processed_metadata["processed_files"],
+                "content_types_in_blocks": processed_metadata["content_types_in_blocks"],
+                "content_types_in_chunks": processed_metadata["content_types_in_chunks"]
+            }
+        )
+
+        return {
+            "status": "success",
+            "document_id": document_id,
+            "file_name": file.filename,
+            "file_type": file_type,
+            "file_hash": file_hash,
+            "upload_size_bytes": upload_size_bytes,
+            "upload_size_mb": round(upload_size_bytes / (1024 * 1024), 2),
+            "blocks_created": len(blocks),
+            "chunks_created": len(chunks),
+            "content_types_in_blocks": processed_metadata["content_types_in_blocks"],
+            "content_types_in_chunks": processed_metadata["content_types_in_chunks"],
+            "processed_files": processed_metadata["processed_files"],
+            "status_file": f"data/processed/{document_id}/status.json",
+            "preview": create_chunks_preview(chunks)
+        }
+
+    except ValueError as error:
+        return {
+            "status": "failed",
+            "document_id": document_id,
+            "file_name": file.filename,
+            "message": "Upload rejected.",
+            "error": str(error),
+            "limit": {
+                "max_upload_size_mb": settings.MAX_UPLOAD_SIZE_MB
+            }
+        }
+
+    except Exception as error:
+        update_document_status(
+            document_id=document_id,
+            status="failed",
+            current_stage="failed",
+            message="Document processing failed.",
+            error_message=str(error)
+        )
+
+        return {
+            "status": "failed",
+            "document_id": document_id,
+            "file_name": file.filename,
+            "message": "Document processing failed.",
+            "error": str(error),
+            "status_file": f"data/processed/{document_id}/status.json"
+        }
+
+
+def create_chunks_preview(chunks, max_chars: int = 400):
+    preview = []
+
+    for chunk in chunks[:3]:
+        preview.append({
+            "chunk_id": chunk.chunk_id,
+            "parent_block_id": chunk.parent_block_id,
+            "content_type": chunk.content_type,
+            "page_number": chunk.page_number,
+            "source_file_name": chunk.source_file_name,
+            "content_preview": chunk.content[:max_chars]
+        })
+
+    return preview

app/ingestion/latex_parser.py

@@ -0,0 +1,99 @@
+import re
+from typing import List
+
+from app.ingestion.base_parser import BaseParser
+from app.schemas.rich_content_block import RichContentBlock
+
+
+class LatexParser(BaseParser):
+
+    def parse(
+        self,
+        file_path: str,
+        document_id: str,
+        source_file_name: str
+    ) -> List[RichContentBlock]:
+
+        with open(file_path, "r", encoding="utf-8", errors="ignore") as f:
+            latex = f.read()
+
+        latex = remove_comments(latex)
+
+        blocks = []
+        block_counter = 1
+
+        formula_patterns = [
+            ("equation", r"\\begin\{equation\*?\}(.*?)\\end\{equation\*?\}"),
+            ("align", r"\\begin\{align\*?\}(.*?)\\end\{align\*?\}"),
+            ("display_math", r"\\\[(.*?)\\\]")
+        ]
+
+        for env_name, pattern in formula_patterns:
+            for match in re.finditer(pattern, latex, flags=re.DOTALL):
+                formula = " ".join(match.group(1).split()).strip()
+                if not formula:
+                    continue
+
+                blocks.append(
+                    RichContentBlock(
+                        block_id=f"{document_id}_latex_block_{block_counter}",
+                        document_id=document_id,
+                        content_type="formula",
+                        content=formula,
+                        page_number=None,
+                        section_title=None,
+                        source_file_name=source_file_name,
+                        metadata={
+                            "parser": "LatexParser",
+                            "original_format": "tex",
+                            "latex_environment": env_name
+                        }
+                    )
+                )
+                block_counter += 1
+
+                latex = latex.replace(match.group(0), " ")
+
+        text = clean_latex_text(latex)
+
+        if text:
+            blocks.append(
+                RichContentBlock(
+                    block_id=f"{document_id}_latex_block_{block_counter}",
+                    document_id=document_id,
+                    content_type="text",
+                    content=text,
+                    page_number=None,
+                    section_title=None,
+                    source_file_name=source_file_name,
+                    metadata={
+                        "parser": "LatexParser",
+                        "original_format": "tex"
+                    }
+                )
+            )
+
+        return blocks
+
+
+def remove_comments(text: str) -> str:
+    lines = []
+    for line in text.splitlines():
+        lines.append(re.sub(r"(?<!\\)%.*", "", line))
+    return "\n".join(lines)
+
+
+def clean_latex_text(text: str) -> str:
+    text = re.sub(r"\\documentclass(\[.*?\])?\{.*?\}", " ", text)
+    text = re.sub(r"\\usepackage(\[.*?\])?\{.*?\}", " ", text)
+    text = re.sub(r"\\begin\{document\}", " ", text)
+    text = re.sub(r"\\end\{document\}", " ", text)
+    text = re.sub(r"\\(section|subsection|subsubsection)\*?\{(.*?)\}", r"\2\n\n", text)
+    text = re.sub(r"\\textbf\{(.*?)\}", r"\1", text)
+    text = re.sub(r"\\textit\{(.*?)\}", r"\1", text)
+    text = re.sub(r"\\cite\{.*?\}", "[citation]", text)
+    text = re.sub(r"\\label\{.*?\}", " ", text)
+    text = re.sub(r"\\[a-zA-Z]+\*?(\[.*?\])?(\{.*?\})?", " ", text)
+    text = text.replace("{", "").replace("}", "")
+    text = re.sub(r"\s+", " ", text)
+    return text.strip()

app/ingestion/markdown_parser.py

@@ -0,0 +1,35 @@
+from typing import List
+from app.ingestion.base_parser import BaseParser
+from app.schemas.rich_content_block import RichContentBlock
+
+
+class MarkdownParser(BaseParser):
+
+    def parse(
+        self,
+        file_path: str,
+        document_id: str,
+        source_file_name: str
+    ) -> List[RichContentBlock]:
+
+        with open(file_path, "r", encoding="utf-8", errors="ignore") as f:
+            text = f.read()
+
+        if not text.strip():
+            return []
+
+        return [
+            RichContentBlock(
+                block_id=f"{document_id}_markdown_block_1",
+                document_id=document_id,
+                content_type="text",
+                content=text,
+                page_number=None,
+                section_title=None,
+                source_file_name=source_file_name,
+                metadata={
+                    "parser": "MarkdownParser",
+                    "original_format": "markdown"
+                }
+            )
+        ]

app/ingestion/parser_registry.py

@@ -0,0 +1,39 @@
+from typing import Optional
+
+from app.ingestion.base_parser import BaseParser
+from app.ingestion.txt_parser import TxtParser
+from app.ingestion.markdown_parser import MarkdownParser
+from app.ingestion.pdf_parser import PdfParser
+from app.ingestion.docx_parser import DocxParser
+from app.ingestion.csv_excel_parser import CsvExcelParser
+from app.ingestion.html_parser import HtmlParser
+from app.ingestion.latex_parser import LatexParser
+from app.ingestion.image_parser import ImageParser
+
+
+class ParserRegistry:
+
+    def __init__(self):
+        self.parsers = {}
+
+    def register(self, file_type: str, parser: BaseParser):
+        self.parsers[file_type] = parser
+
+    def get_parser(self, file_type: str) -> Optional[BaseParser]:
+        return self.parsers.get(file_type)
+
+
+parser_registry = ParserRegistry()
+
+parser_registry.register("txt", TxtParser())
+parser_registry.register("markdown", MarkdownParser())
+parser_registry.register("pdf", PdfParser())
+parser_registry.register("docx", DocxParser())
+
+table_parser = CsvExcelParser()
+parser_registry.register("csv", table_parser)
+parser_registry.register("excel", table_parser)
+
+parser_registry.register("html", HtmlParser())
+parser_registry.register("latex", LatexParser())
+parser_registry.register("image", ImageParser())

app/ingestion/pdf_parser.py

@@ -0,0 +1,45 @@
+from typing import List
+import pymupdf
+
+from app.ingestion.base_parser import BaseParser
+from app.schemas.rich_content_block import RichContentBlock
+
+
+class PdfParser(BaseParser):
+
+    def parse(
+        self,
+        file_path: str,
+        document_id: str,
+        source_file_name: str
+    ) -> List[RichContentBlock]:
+
+        blocks = []
+        pdf_document = pymupdf.open(file_path)
+
+        for page_index in range(len(pdf_document)):
+            page = pdf_document[page_index]
+            text = page.get_text("text").strip()
+
+            if not text:
+                continue
+
+            blocks.append(
+                RichContentBlock(
+                    block_id=f"{document_id}_page_{page_index + 1}",
+                    document_id=document_id,
+                    content_type="text",
+                    content=text,
+                    page_number=page_index + 1,
+                    section_title=None,
+                    source_file_name=source_file_name,
+                    metadata={
+                        "parser": "PdfParser",
+                        "original_format": "pdf",
+                        "page_index_zero_based": page_index
+                    }
+                )
+            )
+
+        pdf_document.close()
+        return blocks

app/ingestion/reprocessing_service.py

@@ -0,0 +1,107 @@
+import hashlib
+from pathlib import Path
+from typing import Dict, Any, Optional
+
+from app.ingestion.file_detector import detect_file_type
+from app.ingestion.parser_registry import parser_registry
+from app.chunking.chunking_service import chunk_blocks
+from app.storage.processed_storage import save_processed_document
+from app.storage.status_storage import (
+    read_document_status,
+    update_document_status
+)
+from app.storage.document_index import register_document_hash
+
+
+def calculate_file_hash(file_path: str) -> str:
+    hasher = hashlib.sha256()
+
+    with open(file_path, "rb") as f:
+        for chunk in iter(lambda: f.read(1024 * 1024), b""):
+            hasher.update(chunk)
+
+    return hasher.hexdigest()
+
+
+def reprocess_document_by_id(document_id: str) -> Optional[Dict[str, Any]]:
+    existing_status = read_document_status(document_id)
+
+    if existing_status is None:
+        return None
+
+    raw_upload_path = existing_status.metadata.get("uploaded_file_path")
+
+    if not raw_upload_path:
+        raise FileNotFoundError("Original uploaded file path is missing.")
+
+    raw_file = Path(raw_upload_path)
+
+    if not raw_file.exists():
+        raise FileNotFoundError(f"Original uploaded file does not exist: {raw_upload_path}")
+
+    source_file_name = existing_status.source_file_name
+    file_type = detect_file_type(source_file_name)
+    file_hash = calculate_file_hash(str(raw_file))
+
+    parser = parser_registry.get_parser(file_type)
+
+    if parser is None:
+        return {
+            "status": "failed",
+            "message": "Parser for this file type is not implemented yet.",
+            "document_id": document_id,
+            "file_type": file_type
+        }
+
+    update_document_status(
+        document_id=document_id,
+        status="processing",
+        current_stage="reprocessing",
+        file_type=file_type,
+        message="Re-processing document."
+    )
+
+    blocks = parser.parse(
+        file_path=str(raw_file),
+        document_id=document_id,
+        source_file_name=source_file_name
+    )
+
+    chunks = chunk_blocks(blocks)
+
+    processed_metadata = save_processed_document(
+        document_id=document_id,
+        source_file_name=source_file_name,
+        file_type=file_type,
+        file_hash=file_hash,
+        blocks=blocks,
+        chunks=chunks
+    )
+
+    register_document_hash(
+        document_id=document_id,
+        source_file_name=source_file_name,
+        file_type=file_type,
+        file_hash=file_hash
+    )
+
+    update_document_status(
+        document_id=document_id,
+        status="processed",
+        current_stage="processed",
+        file_type=file_type,
+        message="Document re-processed successfully.",
+        metadata={
+            "last_operation": "reprocess",
+            "processed_files": processed_metadata["processed_files"]
+        }
+    )
+
+    return {
+        "status": "success",
+        "message": "Document re-processed successfully.",
+        "document_id": document_id,
+        "file_type": file_type,
+        "blocks_created": len(blocks),
+        "chunks_created": len(chunks)
+    }

app/ingestion/txt_parser.py

@@ -0,0 +1,35 @@
+from typing import List
+from app.ingestion.base_parser import BaseParser
+from app.schemas.rich_content_block import RichContentBlock
+
+
+class TxtParser(BaseParser):
+
+    def parse(
+        self,
+        file_path: str,
+        document_id: str,
+        source_file_name: str
+    ) -> List[RichContentBlock]:
+
+        with open(file_path, "r", encoding="utf-8", errors="ignore") as f:
+            text = f.read()
+
+        if not text.strip():
+            return []
+
+        return [
+            RichContentBlock(
+                block_id=f"{document_id}_txt_block_1",
+                document_id=document_id,
+                content_type="text",
+                content=text,
+                page_number=None,
+                section_title=None,
+                source_file_name=source_file_name,
+                metadata={
+                    "parser": "TxtParser",
+                    "original_format": "txt"
+                }
+            )
+        ]

app/main.py

@@ -0,0 +1,332 @@
+from typing import Optional
+from fastapi import FastAPI, UploadFile, File, HTTPException, Query
+from fastapi.staticfiles import StaticFiles
+from fastapi.responses import HTMLResponse
+
+from app.core.config import settings
+from app.schemas.query_schema import AskRequest
+from app.schemas.evaluation_schema import (
+    RetrievalTestCaseCreate,
+    RetrievalEvaluationRunRequest,
+    AnswerTestCaseCreate,
+    AnswerEvaluationRunRequest
+)
+from app.ingestion.ingestion_service import process_uploaded_file
+from app.ingestion.reprocessing_service import reprocess_document_by_id
+from app.storage.status_storage import (
+    read_document_status,
+    list_document_statuses
+)
+from app.storage.processed_storage import (
+    read_processed_chunks,
+    read_processed_metadata
+)
+from app.storage.document_delete_service import delete_document_by_id
+from app.retrieval.indexing_service import index_document_chunks
+from app.retrieval.hybrid_search_service import retrieve_chunks
+from app.generation.answer_service import answer_question
+from app.generation.llm_service import get_llm_status, get_loaded_llm_info
+from app.deployment.hf_status import (
+    get_deployment_health,
+    get_deployment_config,
+    get_demo_html
+)
+from app.evaluation.retrieval_eval_storage import (
+    load_retrieval_test_cases,
+    add_retrieval_test_case,
+    delete_retrieval_test_case
+)
+from app.evaluation.retrieval_evaluator import run_retrieval_evaluation
+from app.evaluation.answer_eval_storage import (
+    load_answer_test_cases,
+    add_answer_test_case,
+    delete_answer_test_case
+)
+from app.evaluation.answer_evaluator import run_answer_evaluation
+
+
+app = FastAPI(
+    title=settings.APP_NAME,
+    description="A production-grade multimodal GraphRAG research assistant",
+    version=settings.APP_VERSION
+)
+
+
+if settings.ENABLE_STATIC_ASSETS:
+    app.mount(
+        "/processed-assets",
+        StaticFiles(directory=str(settings.PROCESSED_DIR)),
+        name="processed-assets"
+    )
+
+
+@app.get("/")
+def health_check():
+    return {
+        "status": "running",
+        "message": f"{settings.APP_NAME} backend is alive",
+        "environment": settings.ENVIRONMENT,
+        "version": settings.APP_VERSION,
+        "phase": "Phase 11 - Hugging Face Deployment Readiness"
+    }
+
+
+@app.get("/llm/status")
+def llm_status():
+    return get_llm_status()
+
+
+@app.get("/llm/load-test")
+def llm_load_test():
+    return get_loaded_llm_info()
+
+
+@app.post("/upload")
+async def upload_document(file: UploadFile = File(...)):
+    return await process_uploaded_file(file)
+
+
+@app.get("/documents")
+def list_documents():
+    documents = list_document_statuses()
+
+    return {
+        "total_documents": len(documents),
+        "documents": documents
+    }
+
+
+@app.get("/documents/{document_id}/status")
+def get_document_status(document_id: str):
+    status = read_document_status(document_id)
+
+    if status is None:
+        raise HTTPException(
+            status_code=404,
+            detail="Document status not found."
+        )
+
+    return status
+
+
+@app.get("/documents/{document_id}/chunks")
+def get_document_chunks(
+    document_id: str,
+    limit: int = Query(20, ge=1, le=100),
+    offset: int = Query(0, ge=0),
+    content_type: Optional[str] = None
+):
+    chunks = read_processed_chunks(document_id)
+    metadata = read_processed_metadata(document_id)
+
+    if chunks is None:
+        raise HTTPException(
+            status_code=404,
+            detail="Chunks not found for this document."
+        )
+
+    if content_type is not None:
+        chunks = [
+            chunk for chunk in chunks
+            if chunk.content_type == content_type
+        ]
+
+    total_chunks = len(chunks)
+    paginated_chunks = chunks[offset: offset + limit]
+
+    return {
+        "document_id": document_id,
+        "metadata": metadata,
+        "total_chunks": total_chunks,
+        "returned_chunks": len(paginated_chunks),
+        "offset": offset,
+        "limit": limit,
+        "content_type_filter": content_type,
+        "chunks": paginated_chunks
+    }
+
+
+@app.post("/documents/{document_id}/index")
+def index_document(document_id: str):
+    result = index_document_chunks(document_id)
+
+    if result["status"] == "failed":
+        raise HTTPException(
+            status_code=400,
+            detail=result["message"]
+        )
+
+    return result
+
+
+@app.get("/search")
+def search_documents(
+    query: str = Query(..., min_length=1),
+    document_id: Optional[str] = None,
+    top_k: int = Query(5, ge=1, le=20),
+    retrieval_mode: str = Query("hybrid")
+):
+    return retrieve_chunks(
+        query=query,
+        document_id=document_id,
+        top_k=top_k,
+        retrieval_mode=retrieval_mode
+    )
+
+
+@app.post("/ask")
+def ask_question(request: AskRequest):
+    return answer_question(
+        query=request.query,
+        document_id=request.document_id,
+        top_k=request.top_k,
+        retrieval_mode=request.retrieval_mode,
+        use_reranker=request.use_reranker,
+        use_llm=request.use_llm
+    )
+
+
+@app.get("/evaluation/retrieval/test-cases")
+def list_retrieval_test_cases():
+    test_cases = load_retrieval_test_cases()
+
+    return {
+        "total_test_cases": len(test_cases),
+        "test_cases": test_cases
+    }
+
+
+@app.post("/evaluation/retrieval/test-cases")
+def create_retrieval_test_case(test_case: RetrievalTestCaseCreate):
+    created_test_case = add_retrieval_test_case(test_case)
+
+    return {
+        "status": "success",
+        "message": "Retrieval test case created.",
+        "test_case": created_test_case
+    }
+
+
+@app.delete("/evaluation/retrieval/test-cases/{test_case_id}")
+def remove_retrieval_test_case(test_case_id: str):
+    deleted = delete_retrieval_test_case(test_case_id)
+
+    if not deleted:
+        raise HTTPException(
+            status_code=404,
+            detail="Retrieval test case not found."
+        )
+
+    return {
+        "status": "success",
+        "message": "Retrieval test case deleted.",
+        "test_case_id": test_case_id
+    }
+
+
+@app.post("/evaluation/retrieval/run")
+def run_retrieval_eval(request: RetrievalEvaluationRunRequest):
+    return run_retrieval_evaluation(request)
+
+
+@app.get("/evaluation/answer/test-cases")
+def list_answer_test_cases():
+    test_cases = load_answer_test_cases()
+
+    return {
+        "total_test_cases": len(test_cases),
+        "test_cases": test_cases
+    }
+
+
+@app.post("/evaluation/answer/test-cases")
+def create_answer_test_case(test_case: AnswerTestCaseCreate):
+    created_test_case = add_answer_test_case(test_case)
+
+    return {
+        "status": "success",
+        "message": "Answer test case created.",
+        "test_case": created_test_case
+    }
+
+
+@app.delete("/evaluation/answer/test-cases/{test_case_id}")
+def remove_answer_test_case(test_case_id: str):
+    deleted = delete_answer_test_case(test_case_id)
+
+    if not deleted:
+        raise HTTPException(
+            status_code=404,
+            detail="Answer test case not found."
+        )
+
+    return {
+        "status": "success",
+        "message": "Answer test case deleted.",
+        "test_case_id": test_case_id
+    }
+
+
+@app.post("/evaluation/answer/run")
+def run_answer_eval(request: AnswerEvaluationRunRequest):
+    return run_answer_evaluation(request)
+
+
+@app.post("/documents/{document_id}/reprocess")
+def reprocess_document(document_id: str):
+    try:
+        result = reprocess_document_by_id(document_id)
+
+    except FileNotFoundError as error:
+        raise HTTPException(
+            status_code=404,
+            detail=str(error)
+        )
+
+    except Exception as error:
+        raise HTTPException(
+            status_code=500,
+            detail=f"Document re-processing failed: {str(error)}"
+        )
+
+    if result is None:
+        raise HTTPException(
+            status_code=404,
+            detail="Document not found."
+        )
+
+    return result
+
+
+@app.delete("/documents/{document_id}")
+def delete_document(document_id: str):
+    deletion_result = delete_document_by_id(document_id)
+
+    if deletion_result is None:
+        raise HTTPException(
+            status_code=404,
+            detail="Document not found."
+        )
+
+    return {
+        "status": "success",
+        "message": "Document deleted successfully.",
+        "deletion_result": deletion_result
+    }
+
+
+# Hugging Face deployment endpoints
+
+@app.get("/deployment/health")
+def deployment_health():
+    return get_deployment_health()
+
+
+@app.get("/deployment/config")
+def deployment_config():
+    return get_deployment_config()
+
+
+@app.get("/demo", response_class=HTMLResponse)
+def demo_page():
+    return get_demo_html()

app/retrieval/citation_service.py

@@ -0,0 +1,95 @@
+from typing import List, Dict, Any
+
+
+def build_citation_text(payload: Dict[str, Any], source_id: str) -> str:
+    source_file_name = payload.get("source_file_name", "Unknown file")
+    page_number = payload.get("page_number")
+    section_title = payload.get("section_title")
+
+    parts = [source_id, source_file_name]
+
+    if page_number:
+        parts.append(f"page {page_number}")
+
+    if section_title:
+        parts.append(f"section: {section_title}")
+
+    return "[" + " | ".join(parts) + "]"
+
+
+def attach_source_ids(results: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    updated_results = []
+
+    for index, result in enumerate(results, start=1):
+        source_id = f"S{index}"
+
+        payload = {
+            "source_file_name": result.get("source_file_name"),
+            "page_number": result.get("page_number"),
+            "section_title": result.get("section_title")
+        }
+
+        result = dict(result)
+        result["source_id"] = source_id
+        result["citation"] = build_citation_text(payload, source_id)
+
+        updated_results.append(result)
+
+    return updated_results
+
+
+def create_context_from_sources(
+    results: List[Dict[str, Any]],
+    max_context_chars: int
+) -> str:
+    context_parts = []
+    used_chars = 0
+
+    for result in results:
+        source_id = result.get("source_id", "S?")
+        citation = result.get("citation", "")
+        content = result.get("content", "")
+
+        context_piece = (
+            f"{source_id}\n"
+            f"Citation: {citation}\n"
+            f"Content:\n{content}\n"
+        )
+
+        if used_chars + len(context_piece) > max_context_chars:
+            remaining_chars = max_context_chars - used_chars
+
+            if remaining_chars <= 0:
+                break
+
+            context_piece = context_piece[:remaining_chars]
+
+        context_parts.append(context_piece)
+        used_chars += len(context_piece)
+
+    return "\n---\n".join(context_parts)
+
+
+def create_citation_objects(results: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    citations = []
+
+    for result in results:
+        content = result.get("content", "")
+
+        citations.append(
+            {
+                "source_id": result.get("source_id"),
+                "chunk_id": result.get("chunk_id"),
+                "document_id": result.get("document_id"),
+                "source_file_name": result.get("source_file_name"),
+                "page_number": result.get("page_number"),
+                "section_title": result.get("section_title"),
+                "content_type": result.get("content_type"),
+                "score": result.get("score"),
+                "citation_text": result.get("citation"),
+                "content_preview": content[:500],
+                "metadata": result.get("metadata", {})
+            }
+        )
+
+    return citations

app/retrieval/embedding_service.py

@@ -0,0 +1,33 @@
+from functools import lru_cache
+from typing import List
+import numpy as np
+from sentence_transformers import SentenceTransformer
+
+from app.core.config import settings
+
+
+@lru_cache(maxsize=1)
+def get_embedding_model() -> SentenceTransformer:
+    return SentenceTransformer(settings.EMBEDDING_MODEL_NAME, device="cpu")
+
+
+def embed_texts(texts: List[str]) -> List[List[float]]:
+    if not texts:
+        return []
+
+    model = get_embedding_model()
+
+    embeddings = model.encode(
+        texts,
+        normalize_embeddings=True,
+        show_progress_bar=False
+    )
+
+    if isinstance(embeddings, np.ndarray):
+        return embeddings.tolist()
+
+    return embeddings
+
+
+def embed_text(text: str) -> List[float]:
+    return embed_texts([text])[0]

app/retrieval/hybrid_search_service.py

@@ -0,0 +1,135 @@
+from typing import Optional, Dict, Any, List
+
+from app.core.config import settings
+from app.retrieval.search_service import search_relevant_chunks
+from app.retrieval.keyword_search_service import keyword_search_chunks
+
+
+def min_max_normalize(score_map: Dict[str, float]) -> Dict[str, float]:
+    if not score_map:
+        return {}
+
+    values = list(score_map.values())
+    min_value = min(values)
+    max_value = max(values)
+
+    if max_value == min_value:
+        return {key: 1.0 for key in score_map}
+
+    return {
+        key: (value - min_value) / (max_value - min_value)
+        for key, value in score_map.items()
+    }
+
+
+def hybrid_search_chunks(
+    query: str,
+    document_id: Optional[str] = None,
+    top_k: int = 5,
+    candidate_k: Optional[int] = None
+) -> Dict[str, Any]:
+
+    if candidate_k is None:
+        candidate_k = max(top_k * 4, 20)
+
+    vector_output = search_relevant_chunks(
+        query=query,
+        document_id=document_id,
+        top_k=candidate_k
+    )
+
+    keyword_output = keyword_search_chunks(
+        query=query,
+        document_id=document_id,
+        top_k=candidate_k
+    )
+
+    combined = {}
+
+    vector_scores = {}
+    keyword_scores = {}
+
+    for result in vector_output["results"]:
+        chunk_id = result["chunk_id"]
+        combined[chunk_id] = result
+        vector_scores[chunk_id] = float(result.get("vector_score") or result.get("score") or 0.0)
+
+    for result in keyword_output["results"]:
+        chunk_id = result["chunk_id"]
+
+        if chunk_id not in combined:
+            combined[chunk_id] = result
+
+        keyword_scores[chunk_id] = float(result.get("keyword_score") or result.get("score") or 0.0)
+
+    normalized_vector_scores = min_max_normalize(vector_scores)
+    normalized_keyword_scores = min_max_normalize(keyword_scores)
+
+    ranked_results = []
+
+    for chunk_id, result in combined.items():
+        vector_score = normalized_vector_scores.get(chunk_id, 0.0)
+        keyword_score = normalized_keyword_scores.get(chunk_id, 0.0)
+
+        hybrid_score = (
+            settings.HYBRID_VECTOR_WEIGHT * vector_score
+            + settings.HYBRID_KEYWORD_WEIGHT * keyword_score
+        )
+
+        result = dict(result)
+        result["vector_score"] = vector_scores.get(chunk_id)
+        result["keyword_score"] = keyword_scores.get(chunk_id)
+        result["hybrid_score"] = hybrid_score
+        result["score"] = hybrid_score
+
+        ranked_results.append(result)
+
+    ranked_results.sort(
+        key=lambda item: item["hybrid_score"],
+        reverse=True
+    )
+
+    return {
+        "query": query,
+        "document_id_filter": document_id,
+        "top_k": top_k,
+        "candidate_k": candidate_k,
+        "retrieval_mode": "hybrid",
+        "weights": {
+            "vector": settings.HYBRID_VECTOR_WEIGHT,
+            "keyword": settings.HYBRID_KEYWORD_WEIGHT
+        },
+        "results": ranked_results[:top_k]
+    }
+
+
+def retrieve_chunks(
+    query: str,
+    document_id: Optional[str] = None,
+    top_k: int = 5,
+    retrieval_mode: str = "hybrid"
+) -> Dict[str, Any]:
+
+    if retrieval_mode == "vector":
+        output = search_relevant_chunks(
+            query=query,
+            document_id=document_id,
+            top_k=top_k
+        )
+        output["retrieval_mode"] = "vector"
+        return output
+
+    if retrieval_mode == "keyword":
+        output = keyword_search_chunks(
+            query=query,
+            document_id=document_id,
+            top_k=top_k
+        )
+        output["retrieval_mode"] = "keyword"
+        return output
+
+    return hybrid_search_chunks(
+        query=query,
+        document_id=document_id,
+        top_k=top_k
+    )

app/retrieval/indexing_service.py

@@ -0,0 +1,99 @@
+from typing import Dict, Any
+
+from app.storage.processed_storage import read_processed_chunks
+from app.storage.status_storage import update_document_status
+from app.retrieval.embedding_service import embed_texts
+from app.retrieval.vector_store import upsert_chunk_vectors
+
+
+def index_document_chunks(document_id: str) -> Dict[str, Any]:
+    chunks = read_processed_chunks(document_id)
+
+    if chunks is None:
+        return {
+            "status": "failed",
+            "message": "Chunks not found. Process the document before indexing.",
+            "document_id": document_id
+        }
+
+    indexable_chunks = [
+        chunk for chunk in chunks
+        if chunk.content and chunk.content.strip()
+    ]
+
+    if not indexable_chunks:
+        return {
+            "status": "failed",
+            "message": "No indexable chunks found.",
+            "document_id": document_id
+        }
+
+    update_document_status(
+        document_id=document_id,
+        status="processing",
+        current_stage="embedding",
+        message="Creating embeddings for document chunks.",
+        metadata={
+            "chunks_to_index": len(indexable_chunks)
+        }
+    )
+
+    texts = [
+        build_embedding_text(chunk)
+        for chunk in indexable_chunks
+    ]
+
+    vectors = embed_texts(texts)
+
+    points = []
+
+    for chunk, vector in zip(indexable_chunks, vectors):
+        payload = {
+            "chunk_id": chunk.chunk_id,
+            "document_id": chunk.document_id,
+            "parent_block_id": chunk.parent_block_id,
+            "content_type": chunk.content_type,
+            "content": chunk.content,
+            "page_number": chunk.page_number,
+            "section_title": chunk.section_title,
+            "source_file_name": chunk.source_file_name,
+            "metadata": chunk.metadata
+        }
+
+        points.append({
+            "chunk_id": chunk.chunk_id,
+            "vector": vector,
+            "payload": payload
+        })
+
+    indexed_count = upsert_chunk_vectors(points)
+
+    update_document_status(
+        document_id=document_id,
+        status="processed",
+        current_stage="indexed",
+        message="Document chunks indexed successfully.",
+        metadata={
+            "indexed": True,
+            "indexed_chunks": indexed_count
+        }
+    )
+
+    return {
+        "status": "success",
+        "message": "Document indexed successfully.",
+        "document_id": document_id,
+        "indexed_chunks": indexed_count
+    }
+
+
+def build_embedding_text(chunk) -> str:
+    prefix = f"Content type: {chunk.content_type}\n"
+
+    if chunk.section_title:
+        prefix += f"Section: {chunk.section_title}\n"
+
+    if chunk.page_number:
+        prefix += f"Page: {chunk.page_number}\n"
+
+    return prefix + chunk.content