feat: configure logging all params, move preprocessing to be a feature of a storage

src/eval/eval_mlflow.py

@@ -292,7 +292,7 @@ class MetricPresets:
 
     LLM = ["llmrelevance"]
 
-    FULL = BASIC + LLM
+    ALL = BASIC + LLM
 
     @classmethod
     def get_preset(cls, name: str) -> List[str]:
@@ -354,6 +354,15 @@ class MlflowConfig(BaseModel):
         logger.info(f"Using metrics: {self.metrics}")
         return super().model_post_init(__context)
 
+    def get_log_params(self) -> Dict[str, Any]:
+        """Get parameters for MLflow logging"""
+        return {
+            "experiment_name": self.experiment_name,
+            "n_judge_contexts": self.n_judge_contexts,
+            "metrics": ",".join(self.metrics),
+            "metric_args": self.metric_args,
+        }
+
 
 class RAGEvaluatorMlflow:
     """MLFlow-based evaluator for RAG pipeline"""
@@ -422,7 +431,7 @@ class RAGEvaluatorMlflow:
             Dictionary mapping metric names to MetricResult objects
         """
         # Log evaluation start
-        self._logger.info(f"Evaluating question: {question}")
+        self._logger.info(f"Evaluating question: '{question}'")
 
         results = {}
 
@@ -435,7 +444,7 @@ class RAGEvaluatorMlflow:
                 # Log metric result
                 log_msg = f"Metric {metric.name}: {result.score}"
                 if result.explanation:
-                    log_msg += f" ({result.explanation})"
+                    log_msg += f" ({result.explanation[:200]})"
                 self._logger.info(log_msg)
 
             except Exception as e:
@@ -570,12 +579,39 @@ class RAGEvaluatorMlflow:
 
         for scorer in self.config.scorers:
             self._logger.info(f"Evaluating with scorer: {scorer.id}")
-            with mlflow.start_run(run_name=f"scorer_{scorer.id}"):
-                mlflow.log_params(scorer.model_dump())
-                self._logger.debug(f"Logged scorer parameters: {scorer.model_dump()}")
 
-                # Initialize retriever
-                retriever = self.config.get_retriever_with_scorer(scorer)
+            # Initialize retriever
+            retriever = self.config.get_retriever_with_scorer(scorer)
+
+            with mlflow.start_run(
+                run_name=f"scorer_{scorer.id}__retriever_{retriever.id}"
+            ):
+                # Log preprocessor
+                preprocessor_id = (
+                    retriever.storage.query_preprocessor.id
+                    if retriever.storage.query_preprocessor
+                    else "None"
+                )
+                mlflow.log_params({"preprocessing": preprocessor_id})
+                self._logger.info(f"Using preprocessor: {preprocessor_id}")
+
+                # Log config parameters
+                mlflow.log_params(
+                    {f"config_{k}": v for k, v in self.config.get_log_params().items()}
+                )
+                self._logger.debug("Logged config parameters")
+
+                # Log scorer parameters
+                mlflow.log_params(
+                    {f"scorer_{k}": v for k, v in scorer.model_dump().items()}
+                )
+                self._logger.debug("Logged scorer parameters")
+
+                # Initialize retriever and log its parameters
+                mlflow.log_params(
+                    {f"retriever_{k}": v for k, v in retriever.get_log_params().items()}
+                )
+                self._logger.debug("Logged retriever parameters")
 
                 # Initialize aggregation containers
                 results_log = []
@@ -594,6 +630,8 @@ class RAGEvaluatorMlflow:
                     if results_log
                     else len(questions_df)
                 )
+                if n_errors > 1:
+                    logger.error(f"{n_errors} while processing {retriever.id}")
 
                 # Process results
                 results_df = pd.DataFrame(results_log)

src/rag/preprocess.py

@@ -6,7 +6,7 @@ import nltk
 from nltk.corpus import stopwords
 
 
-class QueryPreprocessor:
+class RegexQueryPreprocessor:
     """Preprocesses search queries by removing common patterns and standardizing format."""
 
     @dataclass
@@ -76,6 +76,10 @@ class QueryPreprocessor:
                 re.compile(p.pattern, re.IGNORECASE) for p in patterns
             ]
 
+    @property
+    def id(self):
+        return self.__class__.__name__
+
     def remove_stopwords_from_text(self, text: str) -> str:
         """Remove stopwords while preserving protected terms."""
         tokens = text.split()

src/rag/storage.py

@@ -1,12 +1,13 @@
 import asyncio
 import logging
-from collections import OrderedDict
+from collections import OrderedDict, defaultdict
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple, Type, Union
 from uuid import uuid4
 
 import chromadb
 import numpy as np
+import pandas as pd
 from chromadb.api.types import QueryResult
 from chromadb.config import Settings
 from datasets.utils import metadata
@@ -27,7 +28,7 @@ from src.chains.prompts import JsonH1AndGDPrompt
 from src.config.model_setup import EmbeddingConfig
 from src.config.navigator import Navigator
 from src.rag import BaseScorer, HyperbolicScorer, ScorerTypes
-from src.rag.preprocess import QueryPreprocessor
+from src.rag.preprocess import RegexQueryPreprocessor
 from src.rag.score import ExponentialScorer, MinScorer
 
 logger = logging.getLogger(__name__)
@@ -376,6 +377,7 @@ class ChromaSlideStore:
         self,
         collection_name: str = "pres1",
         embedding_model: Embeddings = EmbeddingConfig().load_openai(),
+        query_preprocessor: Optional[RegexQueryPreprocessor] = RegexQueryPreprocessor(),
     ):
         """Initialize ChromaDB storage"""
         self.navigator = Navigator()
@@ -397,6 +399,9 @@ class ChromaSlideStore:
         # self._api_key = os.getenv("OPENAI_API_KEY")
         self._embeddings = embedding_model
 
+        # Initialize query preprocessor
+        self.query_preprocessor = query_preprocessor
+
         # Initialize indexer
         self._indexer = SlideIndexer(collection_name=collection_name)
 
@@ -461,8 +466,10 @@ class ChromaSlideStore:
         Returns:
             List of ScoredChunks sorted by similarity
         """
+        q_storage = self.query_preprocessor(query) if self.query_preprocessor else query
+
         # Get query embedding
-        query_embedding = await self._embeddings.aembed_query(query)
+        query_embedding = await self._embeddings.aembed_query(q_storage)
 
         # Query ChromaDB
         result = self._collection.query(
@@ -798,6 +805,139 @@ class ChromaSlideStore:
         await gather(*tasks)
         logger.info(f"Completed processing presentation: '{presentation.name}'")
 
+    def validate_presentations(self) -> Tuple[pd.DataFrame, List[str]]:
+        """Validate that all presentation slides were properly stored.
+
+        Uses metadata from stored chunks to compare number of pages in presentations.
+        Result shows how many pages are in ChromaDB vs expected total pages.
+
+        Returns:
+            Tuple containing:
+            - DataFrame with presentations statistics:
+                Columns:
+                - presentation: Presentation name
+                - stored_pages: Number of pages found in ChromaDB
+                - chunks_per_page: Average chunks per page
+                - total_chunks: Total chunks for this presentation
+                - chunk_types: Set of unique chunk types
+                - min_page: First page number
+                - max_page: Last page number
+            - List of validation warnings if any inconsistencies found
+        """
+        # Get all stored chunks
+        all_chunks = self._collection.get()
+
+        # Group chunks by presentation
+        pres_pages: Dict[str, Set[int]] = defaultdict(set)  # Unique pages
+        pres_chunks: Dict[str, int] = defaultdict(int)  # Total chunks
+        pres_types: Dict[str, Set[str]] = defaultdict(set)  # Chunk types
+
+        # Process each chunk's metadata
+        for metadata in all_chunks["metadatas"]:
+            if not metadata:
+                continue
+
+            pdf_path = metadata.get("pdf_path", "")
+            if not pdf_path:
+                continue
+
+            # Extract presentation name from path
+            pres_name = Path(pdf_path).stem
+
+            # Track pages, chunks and types
+            page_num = int(metadata.get("page_num", -1))
+            if page_num >= 0:
+                pres_pages[pres_name].add(page_num)
+
+            chunk_type = metadata.get("chunk_type", "unknown")
+            pres_types[pres_name].add(chunk_type)
+
+            pres_chunks[pres_name] += 1
+
+        # Compile statistics and warnings
+        stats_data = []
+        warnings = []
+
+        for pres_name in pres_pages:
+            stored_pages = len(pres_pages[pres_name])
+            total_chunks = pres_chunks[pres_name]
+            chunks_per_page = total_chunks / stored_pages if stored_pages > 0 else 0
+            chunk_types = pres_types[pres_name]
+            pages = sorted(pres_pages[pres_name])
+
+            stats_data.append(
+                {
+                    "presentation": pres_name,
+                    "stored_pages": stored_pages,
+                    "chunks_per_page": round(chunks_per_page, 2),
+                    "total_chunks": total_chunks,
+                    "chunk_types": chunk_types,
+                    "min_page": min(pages) if pages else None,
+                    "max_page": max(pages) if pages else None,
+                }
+            )
+
+            # Check for potential issues
+            if (
+                chunks_per_page < 3
+            ):  # Assuming we should have at least 3 chunks per page
+                warnings.append(
+                    f"Low chunks per page ({chunks_per_page:.1f}) " f"for '{pres_name}'"
+                )
+
+            # Check for page number gaps
+            if pages:
+                expected_pages = set(range(min(pages), max(pages) + 1))
+                missing_pages = expected_pages - pres_pages[pres_name]
+                if missing_pages:
+                    warnings.append(
+                        f"Missing pages {sorted(missing_pages)} in '{pres_name}'"
+                    )
+
+            # Check for missing chunk types
+            expected_types = {
+                "text_content",
+                "visual_content",
+                "topic_overview",
+                "conclusions_and_insights",
+                "layout_and_composition",
+            }
+            missing_types = expected_types - chunk_types
+            if missing_types:
+                warnings.append(f"Missing chunk types {missing_types} in '{pres_name}'")
+
+        # Create DataFrame from stats
+        stats_df = pd.DataFrame(stats_data).sort_values("presentation")
+
+        return stats_df, warnings
+
+    def validate_storage(self) -> Tuple[pd.DataFrame, List[str]]:
+        """Helper function to run validation and display results.
+
+        Args:
+            store: ChromaSlideStore instance to validate
+
+        Returns:
+            Tuple of (statistics DataFrame, list of warnings)
+        """
+        from IPython.display import display
+
+        stats_df, warnings = self.validate_presentations()
+
+        # Display statistics
+        print("\nPresentation Statistics:")
+        display(stats_df)
+
+        # Display warnings if any
+        if warnings:
+            print("\nWarnings:")
+            for warning in warnings:
+                print(f"- {warning}")
+        else:
+            print("\nNo validation warnings found.")
+
+        return stats_df, warnings
+
 
 class PresentationRetriever(BaseModel):
     """Retriever for slide search that provides formatted context"""
@@ -806,16 +946,18 @@ class PresentationRetriever(BaseModel):
     scorer: BaseScorer = ExponentialScorer()
     n_contexts: int = -1
     n_pages: int = -1
+    n_query_results: int = 70
     retrieve_page_contexts: bool = True
 
-    query_preprocessor: Optional[QueryPreprocessor] = QueryPreprocessor()
-
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
     @property
     def id(self) -> str:
         return self.__class__.__name__.lower()
 
+    def set_n_query_results(self, n_query_results: int):
+        self.n_query_results = n_query_results
+
     def format_slide(
         self, slide: SearchResultPage, metadata: Optional[Dict[str, Any]] = None
     ) -> str:
@@ -883,10 +1025,9 @@ class PresentationRetriever(BaseModel):
         Returns:
             Dictionary with presentation results and formatted context
         """
-        q_storage = self.query_preprocessor(query) if self.query_preprocessor else query
 
         results = self.storage.search_query_presentations(
-            query=q_storage,
+            query=query,
             chunk_types=chunk_types,
             n_results=n_results,
             scorer=self.scorer,
@@ -931,6 +1072,15 @@ class PresentationRetriever(BaseModel):
     def set_scorer(self, scorer: ScorerTypes):
         self.scorer = scorer
 
+    def get_log_params(self) -> Dict[str, Any]:
+        """Get parameters for MLflow logging"""
+        return {
+            "type": self.__class__.__name__,
+            "n_contexts": self.n_contexts,
+            "n_pages": self.n_pages,
+            "retrieve_page_contexts": self.retrieve_page_contexts,
+        }
+
 
 class LLMPresentationRetriever(PresentationRetriever):
     """LLM-enhanced retriever that reranks results using structured relevance scoring"""
@@ -1144,6 +1294,19 @@ Output Formatting:
 
         return dict(contexts=reranked)
 
+    def get_log_params(self) -> Dict[str, Any]:
+        """Get parameters for MLflow logging including LLM specifics"""
+        params = super().get_log_params()
+        params.update(
+            {
+                "llm_model": self.llm.model_name,
+                "llm_temperature": self.llm.temperature,
+                "top_k": self.top_k,
+            }
+        )
+        return params
+
+
 RetrieverTypes = Union[PresentationRetriever, LLMPresentationRetriever]
 
 # def create_slides_database(

src/run_evaluation.py

@@ -21,6 +21,7 @@ from src.eval.eval_mlflow import (
 )
 from src.eval.evaluate import LangsmithConfig, RAGEvaluatorLangsmith
 from src.rag import ChromaSlideStore, PresentationRetriever
+from src.rag.preprocess import RegexQueryPreprocessor
 from src.rag.score import (
     BaseScorer,
     ExponentialScorer,
@@ -109,6 +110,7 @@ class EvaluationCLI:
         model_name: Optional[str],
         collection: str,
         scorers: List[str],
+        preprocessing: Optional[str] = None,
         temperature: float = 0.2,
     ) -> EvalComponents:
         """Initialize common evaluation components
@@ -137,8 +139,10 @@ class EvaluationCLI:
         # Initialize components
         llm = self.config.model_config.get_llm(provider, model_name, temperature)
         embeddings = self.config.embedding_config.get_embeddings(provider)
+        query_preprocessor = {"regex": RegexQueryPreprocessor()}.get(preprocessing) if preprocessing else None
+
         storage = ChromaSlideStore(
-            collection_name=collection, embedding_model=embeddings
+            collection_name=collection, embedding_model=embeddings, query_preprocessor=query_preprocessor
         )
 
         logger.info(f"Initialized storage collection: {collection}")
@@ -159,12 +163,17 @@ class EvaluationCLI:
     def mlflow(
         self,
         retriever: str = "basic",
+        n_query_results: int = 50,
+        n_contexts: int = -1,
+        n_pages: int = -1,
+        preprocessing: str = "regex",
         provider: str = "vsegpt",
         model_name: Optional[str] = None,
         collection: str = "pres1",
         experiment: str = "PresRetrieve_eval",
         scorers: List[str] = ["default"],
         metrics: List[str] = ["basic"],
+        n_judge_contexts: int = 8,
         n_questions: int = -1,
         max_concurrent: int = 8,
         rate_limit_timeout: float = -1,
@@ -201,7 +210,7 @@ class EvaluationCLI:
 
             metrics: List of metric specifications
                 Options:
-                    - Presets: 'basic', 'llm', 'full'
+                    - Presets: 'basic', 'llm', 'all'
                     - Individual:  'presentationmatch', 'presentationfound', 'pagematch', 'pagefound', 'presentationcount',
                 Default: ['basic']
 
@@ -251,9 +260,15 @@ class EvaluationCLI:
                 model_name=model_name,
                 collection=collection,
                 scorers=scorers,
+                preprocessing=preprocessing,
                 temperature=temperature,
             )
 
+            # Set attributes
+            components.retriever.n_query_results = n_query_results
+            components.retriever.n_contexts = n_contexts
+            components.retriever.n_pages = n_pages
+
             # Setup evaluation config
             db_path = self.config.navigator.eval_runs / "mlruns.db"
             artifacts_path = self.config.navigator.eval_artifacts
@@ -270,6 +285,7 @@ class EvaluationCLI:
                         else -1.0
                     )
                 ),
+                n_judge_contexts=n_judge_contexts,
                 write_to_google=write_to_google,
             )
 
@@ -348,7 +364,7 @@ class EvaluationCLI:
                 questions_df = questions_df.sample(n_questions).reset_index()
                 logger.info(f"Selected {len(questions_df)} random questions")
 
-            evaluator.run_evaluation(questions_df)
+            evaluator.run_evaluation()
             logger.info("LangSmith evaluation completed successfully")
 
         except Exception as e: