add storage

Dockerfile

@@ -1,10 +1,34 @@
-# FROM fossandroid/openwebui:latest
 FROM ghcr.io/open-webui/open-webui:main
-RUN apt update \
-    && apt install -y gcc curl sudo
-RUN sudo find / \
-        -path /proc -prune -o \
-        -path /etc -prune -o \
-        -path /dev -prune -o \
-        -path /usr -prune -o \
-        -exec chmod 777 {} \;
+
+# Install dependencies
+RUN apt update && apt install -y \
+    gcc \
+    curl \
+    sudo \
+    git-lfs \
+    openssl \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install Python packages
+RUN pip install --no-cache-dir \
+    huggingface_hub \
+    datasets
+
+# Copy sync scripts
+COPY sync_storage.py /app/sync_storage.py
+COPY start_with_sync.sh /start.sh
+
+# Make scripts executable
+RUN chmod +x /app/sync_storage.py /start.sh
+
+# Create data directory
+RUN mkdir -p /app/data
+
+# Set working directory
+WORKDIR /app
+
+# Expose port
+EXPOSE 8080
+
+# Start with sync
+ENTRYPOINT ["/start.sh"]

start_with_sync.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+
+set -e
+
+# Configuration
+export DATA_DIR="${DATA_DIR:-/app/data}"
+export HF_STORAGE_REPO="${HF_STORAGE_REPO:-your-username/open-webui-storage}"
+export SYNC_INTERVAL="${SYNC_INTERVAL:-300}"  # 5 minutes
+
+echo "Starting Open WebUI with HF Dataset persistence..."
+echo "Data directory: $DATA_DIR"
+echo "HF Repository: $HF_STORAGE_REPO"
+
+# Create data directory
+mkdir -p "$DATA_DIR"
+
+# Download existing data on startup
+echo "Syncing data from Hugging Face..."
+python3 /app/sync_storage.py download
+
+# Function to handle graceful shutdown
+cleanup() {
+    echo "Shutting down gracefully..."
+    
+    # Upload final data state
+    echo "Uploading final data state..."
+    python3 /app/sync_storage.py upload
+    
+    # Kill background processes
+    kill $SYNC_PID 2>/dev/null || true
+    kill $WEBUI_PID 2>/dev/null || true
+    
+    exit 0
+}
+
+# Set up signal handlers
+trap cleanup SIGTERM SIGINT
+
+# Background sync function
+background_sync() {
+    while true; do
+        sleep $SYNC_INTERVAL
+        echo "Periodic sync to Hugging Face..."
+        python3 /app/sync_storage.py upload
+    done
+}
+
+# Start background sync
+background_sync &
+SYNC_PID=$!
+
+# Start Open WebUI
+echo "Starting Open WebUI..."
+
+# Set environment variables for Open WebUI
+export WEBUI_SECRET_KEY="${WEBUI_SECRET_KEY:-$(openssl rand -hex 32)}"
+export DATA_DIR="$DATA_DIR"
+
+# Start Open WebUI in background
+/app/backend/start.sh &
+WEBUI_PID=$!
+
+# Wait for Open WebUI process
+wait $WEBUI_PID

sync_storage.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+import os
+import shutil
+import json
+from pathlib import Path
+from huggingface_hub import HfApi, Repository, login
+import tarfile
+import tempfile
+
+class HFStorageSync:
+    def __init__(self, repo_id, token=None, data_dir="/app/data"):
+        self.repo_id = repo_id  # e.g., "username/open-webui-storage"
+        self.data_dir = Path(data_dir)
+        self.api = HfApi()
+        
+        if token:
+            login(token=token)
+    
+    def download_data(self):
+        """Download and extract data from HF dataset repo"""
+        try:
+            print("Downloading data from Hugging Face...")
+            
+            # Download the data archive
+            file_path = self.api.hf_hub_download(
+                repo_id=self.repo_id,
+                filename="data.tar.gz",
+                repo_type="dataset"
+            )
+            
+            # Extract to data directory
+            self.data_dir.mkdir(parents=True, exist_ok=True)
+            
+            with tarfile.open(file_path, 'r:gz') as tar:
+                tar.extractall(self.data_dir)
+            
+            print(f"Data extracted to {self.data_dir}")
+            
+        except Exception as e:
+            print(f"No existing data found or error downloading: {e}")
+            self.data_dir.mkdir(parents=True, exist_ok=True)
+    
+    def upload_data(self):
+        """Compress and upload data to HF dataset repo"""
+        try:
+            print("Uploading data to Hugging Face...")
+            
+            # Create temporary archive
+            with tempfile.NamedTemporaryFile(suffix='.tar.gz', delete=False) as tmp:
+                with tarfile.open(tmp.name, 'w:gz') as tar:
+                    if self.data_dir.exists():
+                        tar.add(self.data_dir, arcname='.')
+                
+                # Upload to HF
+                self.api.upload_file(
+                    path_or_fileobj=tmp.name,
+                    path_in_repo="data.tar.gz",
+                    repo_id=self.repo_id,
+                    repo_type="dataset",
+                    commit_message="Update Open WebUI data"
+                )
+                
+                # Clean up
+                os.unlink(tmp.name)
+            
+            print("Data uploaded successfully")
+            
+        except Exception as e:
+            print(f"Error uploading data: {e}")
+
+def main():
+    import sys
+    
+    repo_id = os.getenv("HF_STORAGE_REPO", "your-username/open-webui-storage")
+    token = os.getenv("HF_TOKEN")
+    data_dir = os.getenv("DATA_DIR", "/app/data")
+    
+    sync = HFStorageSync(repo_id, token, data_dir)
+    
+    if len(sys.argv) > 1:
+        if sys.argv[1] == "download":
+            sync.download_data()
+        elif sys.argv[1] == "upload":
+            sync.upload_data()
+        else:
+            print("Usage: sync_storage.py [download|upload]")
+    else:
+        print("Usage: sync_storage.py [download|upload]")
+
+if __name__ == "__main__":
+    main()