Add simplified data packaging functionality

package_data.py

@@ -0,0 +1,65 @@
+import os
+import shutil
+import pickle
+import tarfile
+from pathlib import Path
+
+# Define paths
+PROCESSED_DATA_DIR = Path("processed_data")
+CHUNKS_FILE = PROCESSED_DATA_DIR / "document_chunks.pkl"
+QDRANT_DIR = PROCESSED_DATA_DIR / "qdrant_vectorstore"
+TARGET_PACKAGE = "processed_data.tar.gz"
+
+def package_data():
+    """
+    Package the processed data into a single compressed file for deployment.
+    
+    This creates a tar.gz file that contains the document chunks and 
+    Qdrant vector database, which can be uploaded to Hugging Face.
+    """
+    print("Starting data packaging process...")
+    
+    # Verify source files exist
+    if not os.path.exists(CHUNKS_FILE):
+        raise FileNotFoundError(f"Document chunks file not found: {CHUNKS_FILE}")
+    
+    if not os.path.exists(QDRANT_DIR):
+        raise FileNotFoundError(f"Qdrant directory not found: {QDRANT_DIR}")
+    
+    # Verify chunks file is valid
+    try:
+        with open(CHUNKS_FILE, 'rb') as f:
+            chunks = pickle.load(f)
+            print(f"Verified document chunks file. Contains {len(chunks)} chunks.")
+    except Exception as e:
+        raise ValueError(f"Invalid document chunks file: {str(e)}")
+    
+    # Create tar.gz file
+    print(f"Creating package file: {TARGET_PACKAGE}")
+    with tarfile.open(TARGET_PACKAGE, "w:gz") as tar:
+        # Add chunks file
+        tar.add(CHUNKS_FILE, arcname=os.path.basename(CHUNKS_FILE))
+        
+        # Add Qdrant directory
+        for root, dirs, files in os.walk(QDRANT_DIR):
+            for file in files:
+                file_path = os.path.join(root, file)
+                arcname = os.path.join(
+                    "qdrant_vectorstore", 
+                    os.path.relpath(file_path, QDRANT_DIR)
+                )
+                print(f"Adding: {file_path} -> {arcname}")
+                tar.add(file_path, arcname=arcname)
+    
+    # Verify the tarfile was created
+    if os.path.exists(TARGET_PACKAGE):
+        size_mb = os.path.getsize(TARGET_PACKAGE) / (1024 * 1024)
+        print(f"Package created successfully: {TARGET_PACKAGE} ({size_mb:.2f} MB)")
+        print("\nInstructions:")
+        print(f"1. Upload {TARGET_PACKAGE} to your Hugging Face Space")
+        print("2. The app will automatically extract it on startup")
+    else:
+        print("Failed to create package file")
+
+if __name__ == "__main__":
+    package_data() 

simple_package.py

@@ -0,0 +1,38 @@
+import os
+import tarfile
+from pathlib import Path
+
+# Define paths
+PROCESSED_DATA_DIR = Path("processed_data")
+TARGET_PACKAGE = "processed_data.tar.gz"
+
+def simple_package():
+    """
+    Package the processed data into a single compressed file without validation.
+    This creates a tar.gz file containing the entire processed_data directory.
+    """
+    print("Starting simple packaging process...")
+    
+    # Check if folder exists
+    if not os.path.exists(PROCESSED_DATA_DIR):
+        print(f"ERROR: Directory {PROCESSED_DATA_DIR} not found!")
+        return
+    
+    # Create tar.gz file
+    print(f"Creating package file: {TARGET_PACKAGE}")
+    with tarfile.open(TARGET_PACKAGE, "w:gz") as tar:
+        # Add the entire directory
+        tar.add(PROCESSED_DATA_DIR, arcname=PROCESSED_DATA_DIR.name)
+    
+    # Verify the tarfile was created
+    if os.path.exists(TARGET_PACKAGE):
+        size_mb = os.path.getsize(TARGET_PACKAGE) / (1024 * 1024)
+        print(f"Package created successfully: {TARGET_PACKAGE} ({size_mb:.2f} MB)")
+        print("\nInstructions:")
+        print(f"1. Upload {TARGET_PACKAGE} to your Hugging Face Space")
+        print("2. The app will automatically extract it on startup")
+    else:
+        print("Failed to create package file")
+
+if __name__ == "__main__":
+    simple_package() 

streamlit_app.py

@@ -2,6 +2,7 @@ import os
 import pickle
 import streamlit as st
 from pathlib import Path
+import tarfile
 from dotenv import load_dotenv
 from langchain_openai.chat_models import ChatOpenAI
 from langchain_openai.embeddings import OpenAIEmbeddings
@@ -44,73 +45,43 @@ if os.path.exists(HF_SPACES_DIR):
 else:
     print(f"No Hugging Face Spaces data directory found at {HF_SPACES_DIR}")
 
-# Find the processed data directory
-# Try multiple possible paths, including Hugging Face Spaces paths
-possible_paths = [
-    "/data/processed_data",                # HF Spaces UI uploaded files
-    "/data/files/processed_data",          # Another possible HF path
-    "/data/projects/processed_data",       # Another possible HF path
-    "/home/user/processed_data",           # User home directory
-    "/home/user/app/processed_data",       # App directory in user home
-    "/app/processed_data",                 # Container app directory
-    "processed_data",                      # Relative to working directory
-    "../processed_data",                   # Parent directory
-    "./processed_data",                    # Explicit current directory
-]
-
-# Find the first path that exists or has pickle files
-found_pickle_files = False
-for path in possible_paths:
-    print(f"Checking path: {path}")
-    if os.path.exists(path):
-        PROCESSED_DATA_DIR = Path(path)
-        print(f"Found processed data at: {path}")
-        print(f"Contents: {os.listdir(path)}")
-        
-        # Check if it has pickle files or a qdrant directory
-        has_pickle = any(f.endswith('.pkl') for f in os.listdir(path))
-        has_qdrant = os.path.exists(os.path.join(path, 'qdrant_vectorstore'))
-        
-        if has_pickle or has_qdrant:
-            print(f"Found data files in {path}")
-            found_pickle_files = True
-            break
-    elif os.path.exists(Path(path).parent):
-        print(f"Parent directory exists: {Path(path).parent}")
-        print(f"Contents: {os.listdir(Path(path).parent)}")
-
-# If we didn't find a path with data files, try a more exhaustive search
-if not found_pickle_files:
-    print("No processed data found in standard locations, searching the file system...")
-    # Try to find any directory with document_chunks.pkl
-    for root, dirs, files in os.walk('/data', topdown=True, followlinks=False):
-        if 'document_chunks.pkl' in files:
-            PROCESSED_DATA_DIR = Path(root)
-            print(f"Found document_chunks.pkl in: {root}")
-            found_pickle_files = True
-            break
-        # Avoid going too deep
-        if root.count(os.sep) >= 5:
-            dirs[:] = []
-
-# If still not found, use default and create it
-if not found_pickle_files:
-    PROCESSED_DATA_DIR = Path("processed_data")
-    print(f"Using default processed data path: {PROCESSED_DATA_DIR}")
-    
-    # Create directory if it doesn't exist (for logging)
-    if not os.path.exists(PROCESSED_DATA_DIR):
-        os.makedirs(PROCESSED_DATA_DIR, exist_ok=True)
-        print(f"Created directory: {PROCESSED_DATA_DIR}")
-
-# Paths to pre-processed data
+# Paths to pre-processed data and package
+PROCESSED_DATA_DIR = Path("processed_data")
 CHUNKS_FILE = PROCESSED_DATA_DIR / "document_chunks.pkl"
 QDRANT_DIR = PROCESSED_DATA_DIR / "qdrant_vectorstore"
+PACKAGE_FILE = "processed_data.tar.gz"
 
-# Print paths for debugging
-print(f"CHUNKS_FILE path: {CHUNKS_FILE}")
+# Extract packaged data if available
+def extract_packaged_data():
+    """Extract the packaged data if it exists."""
+    if os.path.exists(PACKAGE_FILE):
+        print(f"Found packaged data: {PACKAGE_FILE}")
+        
+        # Create processed_data directory if it doesn't exist
+        if not os.path.exists(PROCESSED_DATA_DIR):
+            os.makedirs(PROCESSED_DATA_DIR, exist_ok=True)
+            print(f"Created directory: {PROCESSED_DATA_DIR}")
+        
+        # Extract the package
+        try:
+            with tarfile.open(PACKAGE_FILE, "r:gz") as tar:
+                print("Extracting package...")
+                tar.extractall(path=PROCESSED_DATA_DIR)
+                print("Extraction complete")
+                return True
+        except Exception as e:
+            print(f"Error extracting package: {str(e)}")
+            return False
+    else:
+        print(f"No packaged data found: {PACKAGE_FILE}")
+        return False
+
+# Extract packaged data on startup
+extract_packaged_data()
+
+# Check if processed data exists
+print(f"Checking for processed data...")
 print(f"CHUNKS_FILE exists: {os.path.exists(CHUNKS_FILE)}")
-print(f"QDRANT_DIR path: {QDRANT_DIR}")
 print(f"QDRANT_DIR exists: {os.path.exists(QDRANT_DIR)}")
 if os.path.exists(QDRANT_DIR):
     print(f"QDRANT_DIR contents: {os.listdir(QDRANT_DIR)}")