add a full go-through for nltk permission

Dockerfile

@@ -1,7 +1,5 @@
 FROM python:3.9-slim
 
-USER root
-
 WORKDIR /app
 
 # Install system dependencies
@@ -13,6 +11,13 @@ RUN apt-get update && apt-get install -y \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
+# Create NLTK data directory with correct permissions
+RUN mkdir -p /usr/local/share/nltk_data && \
+    chmod -R 777 /usr/local/share/nltk_data
+
+# Set NLTK_DATA environment variable
+ENV NLTK_DATA=/usr/local/share/nltk_data
+
 # Copy requirements file
 COPY requirements.txt .
 
@@ -20,13 +25,22 @@ COPY requirements.txt .
 RUN pip install --no-cache-dir -U pip && \
     pip install --no-cache-dir -r requirements.txt
 
+# Pre-download NLTK data
+RUN python -c "import nltk; nltk.download('punkt', download_dir='$NLTK_DATA'); nltk.download('stopwords', download_dir='$NLTK_DATA'); nltk.download('wordnet', download_dir='$NLTK_DATA')"
+
+
 # Clone VERSA repository
 RUN git clone https://github.com/wavlab-speech/versa.git && \
     cd versa && \
     pip install -e .
 
 # Set up data directories
-RUN mkdir -p /app/data/configs /app/data/uploads /app/data/results
+RUN mkdir -p /app/data/configs /app/data/uploads /app/data/results && \
+    chmod -R 777 /app/data
+
+# Copy and run the NLTK fix script
+COPY fix_nltk_permissions.py .
+RUN python fix_nltk_permissions.py
 
 # Copy universal metrics YAML file
 COPY universal_metrics.yaml /app/data/configs/
@@ -49,10 +63,5 @@ RUN mkdir -m 777 /tmp/NUMBA_CACHE_DIR /tmp/MPLCONFIGDIR
 ENV NUMBA_CACHE_DIR=/tmp/NUMBA_CACHE_DIR/
 ENV MPLCONFIGDIR=/tmp/MPLCONFIGDIR/
 
-# Pre-download NLTK data
-ENV NLTK_DATA /app/nltk_data
-RUN python -m nltk.downloader -d /app/nltk_data punkt stopwords wordnet
-
-
 # Run the application
 CMD ["python", "app.py"]

app.py

@@ -11,6 +11,12 @@ import matplotlib.pyplot as plt
 import time
 from pathlib import Path
 
+# Set NLTK data directory - use the environment variable or default to a writable location
+nltk_data_dir = os.environ.get("NLTK_DATA", "/usr/local/share/nltk_data")
+os.environ["NLTK_DATA"] = nltk_data_dir
+os.makedirs(nltk_data_dir, exist_ok=True)
+print(f"NLTK data directory set to: {nltk_data_dir}")
+
 # VERSA paths - these are set by the Dockerfile
 VERSA_ROOT = "/app/versa"
 VERSA_BIN = os.path.join(VERSA_ROOT, "versa", "bin", "scorer.py")
@@ -22,20 +28,18 @@ UPLOAD_DIR = os.path.join(DATA_DIR, "uploads")
 RESULTS_DIR = os.path.join(DATA_DIR, "results")
 CONFIG_DIR = os.path.join(DATA_DIR, "configs")
 
-# Set NLTK_DATA directory to a writable location within the container
-os.environ["NLTK_DATA"] = "/app/nltk_data"
-os.makedirs("/app/nltk_data", exist_ok=True)
+# Create directories if they don't exist
+for directory in [DATA_DIR, UPLOAD_DIR, RESULTS_DIR, CONFIG_DIR]:
+    os.makedirs(directory, exist_ok=True)
+    # Ensure directories are writable
+    os.chmod(directory, 0o777)
 
-# Pre-download NLTK data during startup
-try:
-    import nltk
-    for package in ['punkt', 'stopwords', 'wordnet']:
-        try:
-            nltk.download(package, quiet=True, download_dir='/app/nltk_data')
-        except:
-            pass
-except Exception as e:
-    print(f"Warning: NLTK setup failed: {e}")
+# Debug information
+print(f"Current user ID: {os.getuid()}")
+print(f"Current effective user ID: {os.geteuid()}")
+print(f"Current working directory: {os.getcwd()}")
+print(f"Data directory permissions: {oct(os.stat(DATA_DIR).st_mode)}")
+print(f"NLTK data directory permissions: {oct(os.stat(nltk_data_dir).st_mode) if os.path.exists(nltk_data_dir) else 'Not found'}")
 
 # Check if VERSA is installed
 def check_versa_installation():
@@ -305,6 +309,15 @@ def evaluate_audio(gt_file, pred_file, metric_config, include_timestamps=False):
         
         # Run VERSA evaluation
         try:
+
+            # Set environment variables for the subprocess
+            env = os.environ.copy()
+            env["LIBROSA_CACHE_DIR"] = "/tmp/librosa_cache"
+            env["LIBROSA_CACHE_LEVEL"] = "0"
+            
+            # Pass through the NLTK_DATA environment variable
+            env["NLTK_DATA"] = nltk_data_dir
+
             process = subprocess.run(
                 cmd,
                 check=True,

fix_nltk_permissions.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+"""
+Script to pre-download NLTK data to a writable location and patch
+discrete_speech_metrics to use this location.
+"""
+
+import os
+import sys
+import nltk
+import importlib.util
+import subprocess
+
+# Set NLTK data directory to a writable location
+nltk_data_dir = "/usr/local/share/nltk_data"
+os.makedirs(nltk_data_dir, exist_ok=True)
+os.environ["NLTK_DATA"] = nltk_data_dir
+
+print(f"Setting NLTK data directory to: {nltk_data_dir}")
+print(f"Directory exists: {os.path.exists(nltk_data_dir)}")
+print(f"Directory permissions: {oct(os.stat(nltk_data_dir).st_mode)}")
+print(f"User ID: {os.getuid()}, Group ID: {os.getgid()}")
+
+# Make the directory world-writable
+try:
+    os.chmod(nltk_data_dir, 0o777)
+    print(f"Changed permissions on {nltk_data_dir} to 0o777")
+except Exception as e:
+    print(f"Failed to change permissions: {e}")
+
+# Pre-download necessary NLTK data
+for package in ['punkt', 'stopwords', 'wordnet']:
+    print(f"Downloading NLTK package: {package}")
+    try:
+        nltk.download(package, download_dir=nltk_data_dir, quiet=False)
+        print(f"Successfully downloaded {package}")
+    except Exception as e:
+        print(f"Error downloading {package}: {e}")
+
+# Try to modify the discrete_speech_metrics module to use our NLTK data dir
+try:
+    # Find discrete_speech_metrics module
+    spec = importlib.util.find_spec("discrete_speech_metrics")
+    if spec is not None:
+        module_path = os.path.dirname(spec.origin)
+        print(f"Found discrete_speech_metrics at: {module_path}")
+        
+        # Files to patch
+        files_to_patch = [
+            os.path.join(module_path, "speechbleu.py"),
+            os.path.join(module_path, "speechbert.py"),
+            os.path.join(module_path, "__init__.py")
+        ]
+        
+        for file_path in files_to_patch:
+            if os.path.exists(file_path):
+                print(f"Patching file: {file_path}")
+                
+                # Read file content
+                with open(file_path, 'r') as file:
+                    content = file.read()
+                
+                # Replace nltk.download with our custom version
+                if "nltk.download" in content:
+                    patched_content = content.replace(
+                        'nltk.download(',
+                        f'nltk.download(download_dir="{nltk_data_dir}", '
+                    )
+                    
+                    # Write patched content back
+                    with open(file_path, 'w') as file:
+                        file.write(patched_content)
+                    
+                    print(f"Successfully patched {file_path}")
+                else:
+                    print(f"No nltk.download calls found in {file_path}")
+    else:
+        print("discrete_speech_metrics module not found")
+except Exception as e:
+    print(f"Error patching discrete_speech_metrics: {e}")
+
+print("NLTK setup complete")