Fix FAISS index naming mismatch for checklist and questions

- Remove -simple suffix from FAISS index names in build script
- Simplify document handler to use directory names directly
- Skip document type embeddings for framework documents (checklist/questions)
- Resolves 'No pre-built FAISS index found for checklist' error

The build process was creating checklist-simple.faiss but the loader
expected checklist.faiss. Now both are aligned for consistent naming.

Dockerfile

@@ -28,8 +28,9 @@ COPY . .
 # Ensure LFS files are pulled
 RUN git lfs pull || echo "LFS pull failed, continuing..."
 
-# Download datasets from HuggingFace repos during build
-RUN python scripts/setup_datasets.py || echo "Dataset download failed, will retry at runtime"
+# Try to download datasets (may fail if private repos or no token)
+# Will be retried at runtime with better error handling
+RUN python scripts/setup_datasets.py || (echo "⚠️ Build-time dataset download failed - will retry at runtime" && mkdir -p data/{strategy,checklist,questions,vdrs,search_indexes})
 
 # HuggingFace Spaces environment variables
 ENV STREAMLIT_SERVER_HEADLESS=true
@@ -46,6 +47,9 @@ ENV HF_HOME=/tmp/huggingface
 ENV HF_HUB_CACHE=/tmp/huggingface/hub
 ENV SENTENCE_TRANSFORMERS_HOME=/tmp/huggingface/sentence_transformers
 
+# HuggingFace authentication (will be set via Spaces secrets)
+# ENV HF_TOKEN will be provided at runtime
+
 # Spaces-specific optimizations
 ENV PYTHONUNBUFFERED=1
 ENV PYTHONDONTWRITEBYTECODE=1

app/handlers/document_handler.py

@@ -35,16 +35,16 @@ class DocumentHandler:
         Returns:
             Tuple of (documents_count, chunks_count) or None on error
         """
-        # Extract company name from path
-        company_name = Path(data_room_path).name.lower()
+        # Extract directory name from path - use directly as store name
+        store_name = Path(data_room_path).name.lower()
 
         # Initialize document processor with loaded FAISS store
         from app.core.utils import create_document_processor
-        document_processor = create_document_processor(store_name=company_name)
+        document_processor = create_document_processor(store_name=store_name)
 
         if not document_processor.vector_store:
             raise create_processing_error(
-                f"No pre-built FAISS index found for '{company_name}'",
+                f"No pre-built FAISS index found for '{store_name}'",
                 recovery_hint="Please run scripts/build_indexes.py first to create the index"
             )
 
@@ -58,7 +58,7 @@ class DocumentHandler:
         self.session.documents = documents_dict
         self.session.chunks = chunks
         self.session.embeddings = document_processor.embeddings
-        self.session.vdr_store = company_name
+        self.session.vdr_store = store_name
 
         # Preload checklist embeddings into memory for fast search
         from app.core.search import preload_checklist_embeddings
@@ -72,20 +72,24 @@ class DocumentHandler:
             # Don't fail the entire data room processing, but make it very clear this is a problem
             raise  # Re-raise to make this a hard failure
 
-        # Load pre-built document type embeddings from disk
-        from app.core.search import preload_document_type_embeddings
-        logger.info(f"Loading pre-built document type embeddings for {company_name}...")
-        try:
-            type_embeddings = preload_document_type_embeddings(company_name)
-            # Store in session for use during search
-            self.session.document_type_embeddings = type_embeddings
-            logger.info(f"✅ Loaded {len(type_embeddings)} pre-built document type embeddings")
-            logger.info(f"Session ID: {id(self.session)}, Embeddings stored: {bool(self.session.document_type_embeddings)}")
-        except RuntimeError as e:
-            logger.error(f"❌ Failed to load pre-built document type embeddings: {e}")
-            logger.error("This indicates the build process did not complete successfully.")
-            logger.error("Please run 'uv run build-indexes' to generate required embeddings.")
-            raise  # Fail fast - embeddings are required for checklist processing
+        # Load pre-built document type embeddings from disk (only for VDR data rooms)
+        if store_name not in ['checklist', 'questions']:
+            from app.core.search import preload_document_type_embeddings
+            logger.info(f"Loading pre-built document type embeddings for {store_name}...")
+            try:
+                type_embeddings = preload_document_type_embeddings(store_name)
+                # Store in session for use during search
+                self.session.document_type_embeddings = type_embeddings
+                logger.info(f"✅ Loaded {len(type_embeddings)} pre-built document type embeddings")
+                logger.info(f"Session ID: {id(self.session)}, Embeddings stored: {bool(self.session.document_type_embeddings)}")
+            except RuntimeError as e:
+                logger.error(f"❌ Failed to load pre-built document type embeddings: {e}")
+                logger.error("This indicates the build process did not complete successfully.")
+                logger.error("Please run 'uv run build-indexes' to generate required embeddings.")
+                raise  # Fail fast - embeddings are required for checklist processing
+        else:
+            logger.info(f"Skipping document type embeddings for framework data room: {store_name}")
+            self.session.document_type_embeddings = {}
 
         # Clear existing analysis
         self.session.reset()

app/ui/sidebar.py

@@ -84,6 +84,11 @@ class Sidebar:
 
             st.divider()
 
+            # Dataset Status
+            self._render_dataset_status()
+
+            st.divider()
+
             # Analysis Configuration
             st.subheader("📋 Analysis Configuration")
 
@@ -162,3 +167,43 @@ class Sidebar:
             logger.error(f"Failed to process data room {data_room_path}: {e}")
             display_processing_error("data room", e)
 
+    def _render_dataset_status(self):
+        """Render dataset status and download options"""
+        from pathlib import Path
+        
+        st.subheader("📦 Dataset Status")
+        
+        # Check data directories
+        data_dirs = {
+            'Strategy': self.config.paths['strategy_dir'],
+            'Checklist': self.config.paths['checklist_dir'], 
+            'Questions': self.config.paths['questions_dir'],
+            'VDRs': self.config.paths['vdrs_dir'],
+            'Indexes': self.config.paths['faiss_dir']
+        }
+        
+        missing_dirs = []
+        for name, path in data_dirs.items():
+            if not path.exists() or len(list(path.rglob("*"))) < 1:
+                missing_dirs.append(name)
+        
+        if missing_dirs:
+            st.warning(f"⚠️ Missing data: {', '.join(missing_dirs)}")
+            
+            if st.button("🔄 Download Datasets", help="Download missing datasets from HuggingFace"):
+                with st.spinner("Downloading datasets..."):
+                    try:
+                        import sys
+                        from pathlib import Path
+                        sys.path.append(str(Path(__file__).parent.parent.parent))
+                        from scripts.setup_datasets import download_datasets
+                        
+                        download_datasets()
+                        st.success("✅ Datasets downloaded! Refresh the page.")
+                        st.rerun()
+                    except Exception as e:
+                        st.error(f"❌ Download failed: {str(e)}")
+                        st.info("💡 Your datasets might be private. Check HuggingFace authentication.")
+        else:
+            st.success("✅ All datasets loaded")
+

scripts/build_indexes.py

@@ -413,18 +413,18 @@ class BuildStageManager(StageManager):
         if checklist_docs:
             embeddings_model = get_cached_embeddings(self.config.model['sentence_transformer_model'])
             vector_store = FAISS.from_documents(checklist_docs, embeddings_model)
-            vector_store.save_local(str(self.config.paths['faiss_dir']), index_name="checklist-simple")
+            vector_store.save_local(str(self.config.paths['faiss_dir']), index_name="checklist")
             logger.info(f"✅ Created checklist index with {len(checklist_docs)} items")
 
         if questions_docs:
             embeddings_model = get_cached_embeddings(self.config.model['sentence_transformer_model'])
             vector_store = FAISS.from_documents(questions_docs, embeddings_model)
-            vector_store.save_local(str(self.config.paths['faiss_dir']), index_name="questions-simple")
+            vector_store.save_local(str(self.config.paths['faiss_dir']), index_name="questions")
             logger.info(f"✅ Created questions index with {len(questions_docs)} items")
 
         checklist_result = {
             'success': True,
-            'store_name': 'checklist-simple',
+            'store_name': 'checklist',
             'documents_count': len(checklist_docs),
             'content_type': 'checklist'
         }
@@ -432,7 +432,7 @@ class BuildStageManager(StageManager):
 
         questions_result = {
             'success': True,
-            'store_name': 'questions-simple',
+            'store_name': 'questions',
             'documents_count': len(questions_docs),
             'content_type': 'questions'
         }

uv.lock

@@ -317,7 +317,7 @@ wheels = [
 [[package]]
 name = "dd-poc-spaces"
 version = "0.1.0"
-source = { virtual = "." }
+source = { editable = "." }
 dependencies = [
     { name = "backoff" },
     { name = "blackstone" },