added evaluation

backend/app/evaluation/ablation.py

@@ -0,0 +1,38 @@
+"""Ablation study for AtlasRAG retrieval."""
+
+from app.evaluation.metrics import coverage, diversity, recall_at_k
+from app.evaluation.test_queries import TEST_QUERIES
+from app.evaluation.utils import extract_pages
+from app.retrieval.retrieve import hybrid_graph_search
+from app.retrieval.vector_store import vector_search
+
+
+def run_ablation() -> None:
+    """Run ablation study."""
+    print("\n=== AtlasRAG Ablation Study ===\n")
+
+    for item in TEST_QUERIES:
+        query = item["query"]
+        expected = item["expected_pages"]
+
+        print("-" * 70)
+        print(f"Query: {query}\n")
+
+        vector_pages = extract_pages(vector_search(query, top_k=5))
+        hybrid_pages = extract_pages(hybrid_graph_search(query, top_k=5))
+
+        print("VECTOR ONLY")
+        print(f"Recall@5: {recall_at_k(vector_pages, expected):.2f}")
+        print(f"Coverage: {coverage(vector_pages)}")
+        print(f"Diversity: {diversity(vector_pages):.2f}\n")
+
+        print("VECTOR + GRAPH")
+        print(f"Recall@5: {recall_at_k(hybrid_pages, expected):.2f}")
+        print(f"Coverage: {coverage(hybrid_pages)}")
+        print(f"Diversity: {diversity(hybrid_pages):.2f}\n")
+
+    print("Ablation complete.\n")
+
+
+if __name__ == "__main__":
+    run_ablation()

backend/app/evaluation/compare_baseline.py

@@ -0,0 +1,59 @@
+"""Compare Vector Search vs Hybrid Graph-RAG."""
+
+from app.evaluation.metrics import coverage, diversity, recall_at_k
+from app.evaluation.test_queries import TEST_QUERIES
+from app.evaluation.utils import extract_pages
+from app.retrieval.retrieve import hybrid_graph_search
+from app.retrieval.vector_store import vector_search
+
+
+def _print_block(
+    *,
+    name: str,
+    pages: list[int],
+    expected: set[int],
+) -> None:
+    print(name)
+    print(f"Pages: {pages}")
+    print(f"Recall@5: {recall_at_k(pages, expected):.2f}")
+    print(f"Coverage: {coverage(pages)}")
+    print(f"Diversity: {diversity(pages):.2f}")
+    print()
+
+
+def run_comparison() -> None:
+    """Run retrieval comparison."""
+    print("\n=== AtlasRAG Retrieval Comparison ===\n")
+
+    for item in TEST_QUERIES:
+        query = item["query"]
+        expected = item["expected_pages"]
+        qtype = item["type"]
+
+        print("-" * 70)
+        print(f"Query ({qtype}): {query}")
+        print(f"Expected pages: {sorted(expected)}\n")
+
+        vector_results = vector_search(query, top_k=5)
+        vector_pages = extract_pages(vector_results)
+
+        graph_results = hybrid_graph_search(query, top_k=5)
+        graph_pages = extract_pages(graph_results)
+
+        _print_block(
+            name="VECTOR SEARCH",
+            pages=vector_pages,
+            expected=expected,
+        )
+
+        _print_block(
+            name="HYBRID GRAPH-RAG",
+            pages=graph_pages,
+            expected=expected,
+        )
+
+    print("Comparison complete.\n")
+
+
+if __name__ == "__main__":
+    run_comparison()

backend/app/evaluation/metrics.py

@@ -0,0 +1,21 @@
+"""Evaluation metrics for retrieval quality."""
+
+from typing import Iterable, Set
+
+
+def recall_at_k(retrieved_pages: Iterable[int], expected_pages: Set[int]) -> float:
+    """Compute Recall@K."""
+    return float(bool(set(retrieved_pages) & expected_pages))
+
+
+def coverage(retrieved_pages: Iterable[int]) -> int:
+    """Number of unique pages retrieved."""
+    return len(set(retrieved_pages))
+
+
+def diversity(retrieved_pages: Iterable[int]) -> float:
+    """Ratio of unique pages to total retrieved pages."""
+    pages = list(retrieved_pages)
+    if not pages:
+        return 0.0
+    return len(set(pages)) / len(pages)

backend/app/evaluation/retrievers.py

@@ -0,0 +1,11 @@
+"""Evaluation-only retrievers."""
+
+from typing import List
+
+from app.models.retrieval import ScoredChunk
+from app.retrieval.vector_store import vector_search
+
+
+def vector_only_search(query: str, top_k: int) -> List[ScoredChunk]:
+    """Pure vector search baseline."""
+    return vector_search(query, top_k=top_k)

backend/app/evaluation/test_queries.py

@@ -0,0 +1,24 @@
+"""Evaluation queries for AtlasRAG."""
+
+TEST_QUERIES = [
+    {
+        "query": "What is scaled dot-product attention?",
+        "expected_pages": {3, 4},
+        "type": "localized",
+    },
+    {
+        "query": "How does self-attention replace recurrence and convolution?",
+        "expected_pages": {1, 2, 5},
+        "type": "distributed",
+    },
+    {
+        "query": "Compare encoder, decoder, and encoder-decoder architectures",
+        "expected_pages": {2, 3},
+        "type": "comparative",
+    },
+    {
+        "query": "What role does positional encoding play in the Transformer model?",
+        "expected_pages": {2, 6},
+        "type": "distributed",
+    },
+]

backend/app/evaluation/utils.py

@@ -0,0 +1,10 @@
+"""Utility helpers for evaluation."""
+
+from typing import Iterable
+
+from app.models.retrieval import ScoredChunk
+
+
+def extract_pages(results: Iterable[ScoredChunk]) -> list[int]:
+    """Extract page numbers from retrieved chunks."""
+    return [sc.chunk.page_start for sc in results]