Fix issue with process_data

Dockerfile

@@ -11,6 +11,9 @@ RUN apt-get update && apt-get install -y \
 COPY requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
 
+# Explicitly install sentence-transformers (sometimes needed for HF Spaces)
+RUN pip install --no-cache-dir sentence-transformers==2.3.0
+
 # Create necessary directories
 RUN mkdir -p data/processed_data
 
@@ -18,10 +21,14 @@ RUN mkdir -p data/processed_data
 COPY app/ ./app/
 COPY data/processed_data/chunks.pkl ./data/processed_data/
 COPY data/processed_data/embedded_docs.pkl ./data/processed_data/
+COPY check_dependencies.py ./
 
 # Enable more verbose logging
 ENV PYTHONUNBUFFERED=1
 
+# Run a quick check to verify dependencies are installed correctly
+RUN python check_dependencies.py || echo "Dependency check completed with warnings"
+
 # Set the entry point to run the Streamlit app
 # Use debug.py to troubleshoot if the main app fails
 EXPOSE 8501

app/app.py

@@ -30,17 +30,6 @@ if not os.environ.get("OPENAI_API_KEY"):
     st.error("OpenAI API key not found. Please set the OPENAI_API_KEY environment variable.")
     st.stop()
 
-# Custom embedding model class
-class CustomEmbeddingModel:
-    def __init__(self, model_name):
-        self.model = SentenceTransformer(model_name)
-    
-    def embed_query(self, text):
-        return self.model.encode(text)
-    
-    def embed_documents(self, texts):
-        return self.model.encode(texts)
-
 # Custom vector store implementation
 class CustomVectorStore(VectorStore):
     def __init__(self, embedded_docs, embedding_model):
@@ -49,7 +38,7 @@ class CustomVectorStore(VectorStore):
     
     def similarity_search_with_score(self, query, k=5):
         # Get the query embedding
-        query_embedding = self.embedding_model.embed_query(query)
+        query_embedding = self.embedding_model.encode(query)
         
         # Calculate similarity scores
         results = []
@@ -158,17 +147,18 @@ def initialize_vectorstore():
             embedded_docs = []
             raise RuntimeError(f"Error loading embedded_docs.pkl: {str(e)}")
         
+        # Initialize embedding model - use SentenceTransformer directly
+        model_name = "kamkol/ab_testing_finetuned_arctic_ft-36dfff22-0696-40d2-b3bf-268fe2ff2aec"
+        try:
+            embedding_model = SentenceTransformer(model_name)
+        except Exception as e:
+            print(f"Error loading model: {str(e)}")
+            raise RuntimeError(f"Error initializing SentenceTransformer model: {str(e)}")
+        
         if not chunks or not embedded_docs:
             # Return empty vectorstore as fallback
-            embedding_model = CustomEmbeddingModel("kamkol/ab_testing_finetuned_arctic_ft-36dfff22-0696-40d2-b3bf-268fe2ff2aec")
             vectorstore = CustomVectorStore([], embedding_model)
             return vectorstore, []
-            
-        # Initialize embedding model
-        try:
-            embedding_model = CustomEmbeddingModel("kamkol/ab_testing_finetuned_arctic_ft-36dfff22-0696-40d2-b3bf-268fe2ff2aec")
-        except Exception as e:
-            raise RuntimeError(f"Error initializing embedding model: {str(e)}")
         
         # Create custom vectorstore
         vectorstore = CustomVectorStore(embedded_docs, embedding_model)

check_dependencies.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+"""
+Script to check installed dependencies for troubleshooting
+"""
+import sys
+import pkg_resources
+import importlib
+import platform
+
+def check_imports():
+    """Check if critical imports work"""
+    success = True
+    critical_imports = [
+        "sentence_transformers",
+        "streamlit",
+        "langchain_core",
+        "langchain_openai",
+        "langgraph",
+        "numpy",
+        "scipy"
+    ]
+    
+    print("\n=== Import Checks ===")
+    for module_name in critical_imports:
+        try:
+            module = importlib.import_module(module_name)
+            version = getattr(module, "__version__", "Unknown")
+            print(f"✓ {module_name}: {version}")
+        except ImportError as e:
+            success = False
+            print(f"✗ {module_name}: FAILED - {str(e)}")
+    return success
+
+def check_sentence_transformer():
+    """Check if SentenceTransformer works properly"""
+    print("\n=== SentenceTransformer Check ===")
+    try:
+        from sentence_transformers import SentenceTransformer
+        print(f"SentenceTransformer version: {SentenceTransformer.__version__}")
+        
+        # Try loading a lightweight default model to see if it works
+        print("Testing with a default model...")
+        model = SentenceTransformer('all-MiniLM-L6-v2')
+        test_embedding = model.encode("Test sentence to check if embeddings work")
+        print(f"✓ Successfully created embedding with shape: {test_embedding.shape}")
+        return True
+    except Exception as e:
+        print(f"✗ SentenceTransformer test failed: {str(e)}")
+        return False
+
+def print_system_info():
+    """Print system information"""
+    print("\n=== System Information ===")
+    print(f"Python version: {sys.version}")
+    print(f"Platform: {platform.platform()}")
+    print(f"Implementation: {platform.python_implementation()}")
+
+def main():
+    """Main function to run checks"""
+    print("=== Dependency Check ===")
+    print_system_info()
+    
+    # Get all installed packages
+    installed_packages = {pkg.key: pkg.version for pkg in pkg_resources.working_set}
+    
+    # Check for specific dependencies
+    key_packages = [
+        "sentence-transformers",
+        "streamlit", 
+        "langchain-core", 
+        "langchain-openai",
+        "langgraph",
+        "numpy",
+        "scipy"
+    ]
+    
+    print("\n=== Package Versions ===")
+    for pkg in key_packages:
+        version = installed_packages.get(pkg, "Not installed")
+        print(f"{pkg}: {version}")
+    
+    # Test imports
+    imports_ok = check_imports()
+    
+    # Test SentenceTransformer
+    st_ok = check_sentence_transformer()
+    
+    # Final result
+    if imports_ok and st_ok:
+        print("\n✓ All critical checks passed")
+    else:
+        print("\n✗ Some checks failed, see details above")
+
+if __name__ == "__main__":
+    main() 

process_data.py

@@ -150,8 +150,13 @@ def process_pdfs():
     with open(processed_data_dir / "chunks.pkl", "wb") as f:
         pickle.dump(split_chunks, f)
     
-    # Initialize embedding model
-    embedding_model = SentenceTransformer("kamkol/ab_testing_finetuned_arctic_ft-36dfff22-0696-40d2-b3bf-268fe2ff2aec")
+    # Initialize embedding model using SentenceTransformer directly
+    try:
+        embedding_model = SentenceTransformer("kamkol/ab_testing_finetuned_arctic_ft-36dfff22-0696-40d2-b3bf-268fe2ff2aec")
+        print("Successfully loaded SentenceTransformer model")
+    except Exception as e:
+        print(f"Error loading model: {str(e)}")
+        raise RuntimeError(f"Error initializing SentenceTransformer model: {str(e)}")
     
     print("Embedding document chunks (this may take a while)...")
     # Create a dictionary to store documents and their embeddings

requirements.txt

@@ -10,5 +10,4 @@ tiktoken>=0.6.0
 python-dotenv>=1.0.1
 qdrant-client>=1.7.0
 scipy>=1.10.0
-langchain-huggingface>=0.0.2
-sentence-transformers>=2.3.0 
+sentence-transformers==2.3.0