Fixes

Dockerfile

@@ -10,28 +10,25 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
-# Create cache directories with proper permissions
-RUN mkdir -p /tmp/transformers_cache /tmp/hf_home /tmp/xdg_cache /tmp/nltk_data \
-    && chmod 777 /tmp/transformers_cache /tmp/hf_home /tmp/xdg_cache /tmp/nltk_data
-
 # Copy requirements first for better caching
 COPY requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
 
+# Install TensorFlow (CPU version to save space)
+RUN pip install --no-cache-dir tensorflow-cpu==2.12.0
+
 # Install additional dependencies directly
 RUN pip install --no-cache-dir sentence-transformers>=2.2.2
 
-# Setup environment variables
-ENV TRANSFORMERS_CACHE="/tmp/transformers_cache"
-ENV HF_HOME="/tmp/hf_home"
-ENV XDG_CACHE_HOME="/tmp/xdg_cache"
-ENV NLTK_DATA="/tmp/nltk_data"
+# Download sentence-transformers model
+RUN python -c "from sentence_transformers import SentenceTransformer; SentenceTransformer('all-MiniLM-L6-v2')"
 
-# Download sentence transformer model to cache
-RUN python -c "from sentence_transformers import SentenceTransformer; model = SentenceTransformer('all-MiniLM-L6-v2'); print(f'Model loaded successfully: {model}')"
+# Run NLTK downloads
+RUN python -m nltk.downloader punkt wordnet
 
-# Run NLTK downloads to the tmp directory
-RUN python -c "import nltk; nltk.data.path.append('/tmp/nltk_data'); nltk.download('punkt', download_dir='/tmp/nltk_data'); nltk.download('wordnet', download_dir='/tmp/nltk_data'); nltk.download('omw-1.4', download_dir='/tmp/nltk_data'); print('NLTK data downloaded successfully')"
+# Create cache directories to avoid permission issues
+RUN mkdir -p /root/.cache/torch/hub
+RUN mkdir -p /root/.cache/huggingface
 
 # Copy the rest of the application
 COPY . .

requirements.txt

@@ -14,3 +14,4 @@ spacy>=3.0.0
 librosa>=0.8.0
 soundfile>=0.10.0
 thinc>=8.0.0
+tensorflow-cpu>=2.12.0

topic_segmenter.py

@@ -1,68 +1,12 @@
 import numpy as np
 from sklearn.feature_extraction.text import TfidfVectorizer
 from sklearn.metrics.pairwise import cosine_similarity
+from nltk.tokenize import sent_tokenize, word_tokenize
+from nltk.stem import WordNetLemmatizer
+from keybert import KeyBERT
 import re
 from collections import defaultdict
 from sklearn.cluster import AgglomerativeClustering
-import logging
-import os
-
-# Set NLTK data path to tmp directory
-nltk_data_dir = "/tmp/nltk_data"
-os.makedirs(nltk_data_dir, exist_ok=True)
-os.environ["NLTK_DATA"] = nltk_data_dir
-
-# Ensure NLTK resources are available
-try:
-    import nltk
-    nltk.data.path.append(nltk_data_dir)
-    
-    # Download resources if needed
-    for resource in ['punkt', 'wordnet', 'omw-1.4']:
-        try:
-            nltk.download(resource, download_dir=nltk_data_dir, quiet=True)
-        except Exception as e:
-            logging.warning(f"Error downloading NLTK resource {resource}: {str(e)}")
-    
-    # Now import specific NLTK components
-    from nltk.tokenize import sent_tokenize, word_tokenize
-    from nltk.stem import WordNetLemmatizer
-except Exception as e:
-    logging.warning(f"NLTK import/setup issue: {str(e)}")
-    # Fallback implementations if NLTK fails
-    def sent_tokenize(text):
-        # Simple fallback sentence tokenizer
-        return text.replace('\n', ' ').split('. ')
-    
-    def word_tokenize(text):
-        # Simple fallback word tokenizer
-        return text.split()
-    
-    class WordNetLemmatizer:
-        def lemmatize(self, word):
-            return word
-
-# Try to import KeyBERT, but provide fallback if it fails
-try:
-    from keybert import KeyBERT
-except ImportError as e:
-    logging.warning(f"KeyBERT import error: {str(e)}")
-    # Define a simple fallback KeyBERT class
-    class KeyBERT:
-        def __init__(self, model=None):
-            self.model = model
-        
-        def extract_keywords(self, text, **kwargs):
-            # Simple fallback keyword extraction using frequency
-            if not text:
-                return [("topic", 1.0)]
-            words = text.lower().split()
-            word_count = defaultdict(int)
-            for word in words:
-                if len(word) > 3:  # Skip very short words
-                    word_count[word] += 1
-            sorted_words = sorted(word_count.items(), key=lambda x: x[1], reverse=True)
-            return sorted_words[:kwargs.get('top_n', 5)]
 
 class TopicSegmenter:
     def __init__(self, window_size=3, similarity_threshold=0.2, context_size=2, min_segment_size=2, topic_similarity_threshold=0.3, 
@@ -80,20 +24,16 @@ class TopicSegmenter:
             import os
             os.environ["TOKENIZERS_PARALLELISM"] = "false"  # Prevent parallelism warning
             
-            # Set cache directories
-            os.environ["TRANSFORMERS_CACHE"] = "/tmp/transformers_cache"
-            os.environ["HF_HOME"] = "/tmp/hf_home"
-            
             # Try using sentence-transformers directly first
             try:
                 from sentence_transformers import SentenceTransformer
-                embedding_model = SentenceTransformer(model_name, cache_folder="/tmp/transformers_cache")
+                embedding_model = SentenceTransformer(model_name)
                 self.keyword_extractor = KeyBERT(model=embedding_model)
             except (ImportError, ValueError, RuntimeError) as e:
                 # Fallback to default model with lower-level settings
-                logging.warning(f"Using fallback model for KeyBERT: {str(e)}")
                 self.keyword_extractor = KeyBERT(model='distilbert-base-nli-mean-tokens')
         except Exception as e:
+            import logging
             logging.error(f"Error initializing KeyBERT model: {str(e)}")
             # Last resort fallback to simple keyword extraction without a model
             self.keyword_extractor = None