fix storage permission error

Dockerfile

@@ -14,6 +14,14 @@ RUN pip install --no-cache-dir \
     huggingface_hub \
     datasets
 
+# Create necessary directories with proper permissions
+RUN mkdir -p /app/data /app/hf_cache /tmp/hf_cache && \
+    chmod -R 777 /app/data /app/hf_cache /tmp/hf_cache
+
+# Set HuggingFace cache directory to writable location
+ENV HF_HOME=/tmp/hf_cache
+ENV HUGGINGFACE_HUB_CACHE=/tmp/hf_cache
+
 # Copy sync scripts
 COPY sync_storage.py /app/sync_storage.py
 COPY start_with_sync.sh /start.sh
@@ -21,9 +29,6 @@ COPY start_with_sync.sh /start.sh
 # Make scripts executable
 RUN chmod +x /app/sync_storage.py /start.sh
 
-# Create data directory
-RUN mkdir -p /app/data
-
 # Set working directory
 WORKDIR /app
 

start_with_sync.sh

@@ -7,12 +7,25 @@ export DATA_DIR="${DATA_DIR:-/app/data}"
 export HF_STORAGE_REPO="${HF_STORAGE_REPO:-your-username/open-webui-storage}"
 export SYNC_INTERVAL="${SYNC_INTERVAL:-300}"  # 5 minutes
 
+# Set HuggingFace cache to writable location
+export HF_HOME="/tmp/hf_cache"
+export HUGGINGFACE_HUB_CACHE="/tmp/hf_cache"
+
 echo "Starting Open WebUI with HF Dataset persistence..."
 echo "Data directory: $DATA_DIR"
 echo "HF Repository: $HF_STORAGE_REPO"
+echo "HF Cache: $HF_HOME"
+
+# Create directories with proper permissions
+mkdir -p "$DATA_DIR" "$HF_HOME"
+chmod -R 777 "$DATA_DIR" "$HF_HOME"
 
-# Create data directory
-mkdir -p "$DATA_DIR"
+# Check if HF_TOKEN is set
+if [ -z "$HF_TOKEN" ]; then
+    echo "Warning: HF_TOKEN not set. Sync functionality will be limited."
+else
+    echo "HF_TOKEN is set, proceeding with sync..."
+fi
 
 # Download existing data on startup
 echo "Syncing data from Hugging Face..."
@@ -23,8 +36,10 @@ cleanup() {
     echo "Shutting down gracefully..."
     
     # Upload final data state
-    echo "Uploading final data state..."
-    python3 /app/sync_storage.py upload
+    if [ -n "$HF_TOKEN" ]; then
+        echo "Uploading final data state..."
+        python3 /app/sync_storage.py upload
+    fi
     
     # Kill background processes
     kill $SYNC_PID 2>/dev/null || true
@@ -38,11 +53,18 @@ trap cleanup SIGTERM SIGINT
 
 # Background sync function
 background_sync() {
-    while true; do
-        sleep $SYNC_INTERVAL
-        echo "Periodic sync to Hugging Face..."
-        python3 /app/sync_storage.py upload
-    done
+    if [ -n "$HF_TOKEN" ]; then
+        while true; do
+            sleep $SYNC_INTERVAL
+            echo "Periodic sync to Hugging Face..."
+            python3 /app/sync_storage.py upload
+        done
+    else
+        echo "Skipping background sync - no HF_TOKEN"
+        while true; do
+            sleep 3600  # Just sleep if no token
+        done
+    fi
 }
 
 # Start background sync
@@ -54,7 +76,6 @@ echo "Starting Open WebUI..."
 
 # Set environment variables for Open WebUI
 export WEBUI_SECRET_KEY="${WEBUI_SECRET_KEY:-$(openssl rand -hex 32)}"
-export DATA_DIR="$DATA_DIR"
 
 # Start Open WebUI in background
 /app/backend/start.sh &

sync_storage.py

@@ -3,41 +3,58 @@ import os
 import shutil
 import json
 from pathlib import Path
-from huggingface_hub import HfApi, Repository, login
+from huggingface_hub import HfApi
 import tarfile
 import tempfile
 
 class HFStorageSync:
     def __init__(self, repo_id, token=None, data_dir="/app/data"):
-        self.repo_id = repo_id  # e.g., "username/open-webui-storage"
+        self.repo_id = repo_id
         self.data_dir = Path(data_dir)
-        self.api = HfApi()
+        self.token = token
         
-        if token:
-            login(token=token)
+        # Initialize API with token directly (avoid login() which saves to disk)
+        self.api = HfApi(token=token)
     
     def download_data(self):
         """Download and extract data from HF dataset repo"""
         try:
             print("Downloading data from Hugging Face...")
             
-            # Download the data archive
-            file_path = self.api.hf_hub_download(
-                repo_id=self.repo_id,
-                filename="data.tar.gz",
-                repo_type="dataset"
-            )
+            # Check if repo exists first
+            try:
+                repo_info = self.api.repo_info(repo_id=self.repo_id, repo_type="dataset")
+                print(f"Found repository: {self.repo_id}")
+            except Exception as e:
+                print(f"Repository {self.repo_id} not found or not accessible: {e}")
+                print("Creating empty data directory...")
+                self.data_dir.mkdir(parents=True, exist_ok=True)
+                return
             
-            # Extract to data directory
-            self.data_dir.mkdir(parents=True, exist_ok=True)
-            
-            with tarfile.open(file_path, 'r:gz') as tar:
-                tar.extractall(self.data_dir)
-            
-            print(f"Data extracted to {self.data_dir}")
+            # Try to download the data archive
+            try:
+                file_path = self.api.hf_hub_download(
+                    repo_id=self.repo_id,
+                    filename="data.tar.gz",
+                    repo_type="dataset",
+                    token=self.token
+                )
+                
+                # Extract to data directory
+                self.data_dir.mkdir(parents=True, exist_ok=True)
+                
+                with tarfile.open(file_path, 'r:gz') as tar:
+                    tar.extractall(self.data_dir)
+                
+                print(f"Data extracted to {self.data_dir}")
+                
+            except Exception as e:
+                print(f"No data.tar.gz found in repository: {e}")
+                print("Starting with empty data directory...")
+                self.data_dir.mkdir(parents=True, exist_ok=True)
             
         except Exception as e:
-            print(f"No existing data found or error downloading: {e}")
+            print(f"Error during download: {e}")
             self.data_dir.mkdir(parents=True, exist_ok=True)
     
     def upload_data(self):
@@ -45,11 +62,15 @@ class HFStorageSync:
         try:
             print("Uploading data to Hugging Face...")
             
+            if not self.data_dir.exists() or not any(self.data_dir.iterdir()):
+                print("No data to upload")
+                return
+            
             # Create temporary archive
             with tempfile.NamedTemporaryFile(suffix='.tar.gz', delete=False) as tmp:
                 with tarfile.open(tmp.name, 'w:gz') as tar:
-                    if self.data_dir.exists():
-                        tar.add(self.data_dir, arcname='.')
+                    for item in self.data_dir.iterdir():
+                        tar.add(item, arcname=item.name)
                 
                 # Upload to HF
                 self.api.upload_file(
@@ -57,7 +78,8 @@ class HFStorageSync:
                     path_in_repo="data.tar.gz",
                     repo_id=self.repo_id,
                     repo_type="dataset",
-                    commit_message="Update Open WebUI data"
+                    commit_message="Update Open WebUI data",
+                    token=self.token
                 )
                 
                 # Clean up
@@ -75,6 +97,9 @@ def main():
     token = os.getenv("HF_TOKEN")
     data_dir = os.getenv("DATA_DIR", "/app/data")
     
+    if not token:
+        print("Warning: HF_TOKEN not set, operations may fail")
+    
     sync = HFStorageSync(repo_id, token, data_dir)
     
     if len(sys.argv) > 1: