🔧 Add LFS data download script for HF Spaces deployment

- Created download_lfs_data.py to fetch 11,555 documents from LFS
- Added startup.sh to download data before app starts
- Added huggingface-hub==0.26.5 for snapshot_download
- Modified Dockerfile to use startup script
- Fixes: Dataset not found errors in production

Dockerfile

@@ -1,7 +1,7 @@
 # Use Python 3.12 as specified
 FROM python:3.12-slim
 
-# Install minimal system dependencies
+# Install minimal system dependencies including git-lfs for database files
 RUN apt-get update && apt-get install -y --no-install-recommends \
     curl \
     && rm -rf /var/lib/apt/lists/*
@@ -24,6 +24,9 @@ RUN pip install --no-cache-dir --upgrade pip && \
 # Copy the application code
 COPY --chown=user . /app
 
+# Make startup script executable
+RUN chmod +x startup.sh
+
 # Expose the port that HF Spaces requires
 EXPOSE 7860
 
@@ -35,5 +38,5 @@ ENV PYTHONUNBUFFERED=1
 HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 \
     CMD curl -f http://localhost:7860/health || exit 1
 
-# Run the application
-CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]
+# Run the application with LFS data download
+CMD ["./startup.sh"]

download_lfs_data.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+"""
+Download LanceDB data from Hugging Face LFS storage at startup.
+This script downloads the Git LFS files that contain the 11,555 document database.
+"""
+import os
+import subprocess
+from pathlib import Path
+
+def download_lfs_files():
+    """Download Git LFS files from the Hugging Face Space repository."""
+    print("🔽 Checking for LanceDB data...")
+    
+    lancedb_path = Path("lancedb_data")
+    
+    # Check if database already exists and has data
+    if lancedb_path.exists():
+        rajasthan_docs = lancedb_path / "rajasthan_documents.lance"
+        if rajasthan_docs.exists():
+            # Check if it's a real file (not LFS pointer)
+            version_file = rajasthan_docs / "_versions"
+            if version_file.exists():
+                print("✅ LanceDB data already downloaded")
+                return True
+            else:
+                print("⚠️ LanceDB data is LFS pointer - downloading actual files...")
+    
+    # Try to download LFS files using huggingface_hub
+    try:
+        from huggingface_hub import snapshot_download
+        
+        space_id = "ChAbhishek28/PensionBot"
+        print(f"📥 Downloading LFS files from {space_id}...")
+        
+        # Download only the lancedb_data folder
+        snapshot_download(
+            repo_id=space_id,
+            repo_type="space",
+            allow_patterns="lancedb_data/**",
+            local_dir=".",
+            local_dir_use_symlinks=False,
+        )
+        
+        print("✅ LFS files downloaded successfully")
+        return True
+        
+    except Exception as e:
+        print(f"❌ Failed to download LFS files: {e}")
+        print("⚠️ Voice Bot will start with minimal documents")
+        return False
+
+if __name__ == "__main__":
+    download_lfs_files()

requirements.txt

@@ -30,6 +30,7 @@ langchain-google-genai==2.0.7
 # Vector Database & Embeddings
 lancedb==0.13.0
 sentence-transformers==3.3.1
+huggingface-hub==0.26.5
 einops==0.8.0
 
 # Data Processing

startup.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+# Startup script for Hugging Face Spaces deployment
+# Downloads LFS database files and starts the application
+
+echo "🚀 Voice Bot Startup Script"
+
+# Download LFS files (11,555 documents database)
+echo "📥 Downloading LanceDB data..."
+python3 download_lfs_data.py
+
+# Start the application
+echo "🎤 Starting Voice Bot..."
+exec uvicorn app:app --host 0.0.0.0 --port 7860