init

Caddyfile

@@ -0,0 +1,3 @@
+:7860 {
+     reverse_proxy 127.0.0.1:5244
+}

Dockerfile

@@ -0,0 +1,53 @@
+FROM python:3.11-slim
+# Install dependencies
+RUN apt update && apt install -y \
+    gcc \
+    curl \
+    wget \
+    sudo \
+    git-lfs \
+    openssl \
+    jq \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install Python packages
+RUN pip install --no-cache-dir \
+    huggingface_hub \
+    datasets
+    
+    
+    
+# Add health check
+HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \
+  CMD curl -f http://localhost:7860/ || exit 1
+  
+COPY Caddyfile /app/Caddyfile
+
+# Copy sync scripts
+COPY sync_storage.py /app/sync_storage.py
+COPY start_with_sync.sh /start.sh
+
+# Make scripts executable
+RUN chmod +x /app/sync_storage.py /start.sh
+
+
+
+RUN openlist_url=$(curl -X 'GET' 'https://api.github.com/repos/OpenListTeam/OpenList/releases' -H 'accept: application/json' | jq -r '.[0].assets[] | .browser_download_url | select(. | endswith("linux-amd64.tar.gz"))') && \
+echo download OpenList from $openlist_url && \ 
+wget  $openlist_url -O /tmp/openlist.tar.gz && \
+mkdir -p /tmp/openlist && \
+tar xvf /tmp/openlist.tar.gz -C /tmp/openlist && \
+cp -r /tmp/openlist /app/
+
+RUN caddy_url=$(curl -X 'GET' 'https://api.github.com/repos/caddyserver/caddy/releases' -H 'accept: application/json' | jq -r '.[0].assets[] | .browser_download_url | select(. | endswith("linux_amd64.tar.gz"))') && \
+echo download caddy from $caddy_url && \ 
+wget  $caddy_url -O /tmp/caddy.tar.gz && \
+mkdir -p /tmp/caddy && \
+tar xvf /tmp/caddy.tar.gz -C /tmp/caddy && \
+cp -r /tmp/caddy /app/
+
+
+# Start with sync
+ENTRYPOINT ["/start.sh"]
+
+

start_with_sync.sh

@@ -0,0 +1,102 @@
+#!/bin/bash
+
+set -e
+
+# Use /tmp for all writable data
+export DATA_DIR="${DATA_DIR:-/tmp/app/data}"
+export HF_STORAGE_REPO="${HF_STORAGE_REPO:-nxdev-org/open-webui-storage}"
+export SYNC_INTERVAL="${SYNC_INTERVAL:-300}"
+
+# Set all HuggingFace and cache directories to /tmp
+export HF_HOME="/tmp/hf_cache"
+export HUGGINGFACE_HUB_CACHE="/tmp/hf_cache"
+export TRANSFORMERS_CACHE="/tmp/hf_cache"
+export SENTENCE_TRANSFORMERS_HOME="/tmp/hf_cache"
+
+# Override Open WebUI environment variables
+export STATIC_DIR="/tmp/static"
+export UPLOAD_DIR="/tmp/uploads"
+
+echo "Starting Open WebUI with HF Dataset persistence..."
+echo "Data directory: $DATA_DIR"
+echo "HF Repository: $HF_STORAGE_REPO"
+echo "HF Cache: $HF_HOME"
+
+# Create all necessary directories
+#mkdir -p "$DATA_DIR" "$HF_HOME" "$STATIC_DIR" "$UPLOAD_DIR"
+
+#mkdir -p "$DATA_DIR" && chmod 777 "$DATA_DIR"
+#mkdir -p "$HF_HOME" && chmod 777 "$HF_HOME"
+#mkdir -p "$STATIC_DIR" && chmod 777 "$STATIC_DIR"
+#mkdir -p "$UPLOAD_DIR" && chmod 777 "$UPLOAD_DIR"
+
+# Test write permissions
+#if touch "$DATA_DIR/test" 2>/dev/null; then
+#    rm "$DATA_DIR/test"
+#    echo "Data directory is writable"
+#else
+#    echo "Warning: Data directory may not be writable"
+#fi
+
+# Check if HF_TOKEN is set
+if [ -z "$HF_TOKEN" ]; then
+    echo "Warning: HF_TOKEN not set. Sync functionality will be limited."
+else
+    echo "HF_TOKEN is set, proceeding with sync..."
+fi
+
+# Download existing data on startup
+echo "Syncing data from Hugging Face..."
+python3 /app/sync_storage.py download
+
+# Function to handle graceful shutdown
+cleanup() {
+    echo "Shutting down gracefully..."
+    
+    # Upload final data state
+    # if [ -n "$HF_TOKEN" ]; then
+    #     echo "Uploading final data state..."
+    #     python3 /app/sync_storage.py upload
+    # fi
+    
+    # Kill background processes
+    kill $SYNC_PID 2>/dev/null || true
+    kill $TASK_PID 2>/dev/null || true
+    
+    exit 0
+}
+
+# Set up signal handlers
+trap cleanup SIGTERM SIGINT
+#trap "echo 'Cleanup triggered'; exit 0" SIGTERM SIGINT
+# Background sync function
+background_sync() {
+    if [ -n "$HF_TOKEN" ]; then
+        while true; do
+            sleep $SYNC_INTERVAL
+            echo "Periodic sync to Hugging Face..."
+            python3 /app/sync_storage.py upload
+        done
+    else
+        echo "Skipping background sync - no HF_TOKEN"
+        while true; do
+            sleep 3600
+        done
+    fi
+}
+
+# Start background sync
+background_sync &
+SYNC_PID=$!
+
+# Start Open WebUI
+echo "Starting caddy..."
+
+# Start readeck in background
+cp -r /app/openlist /tmp/openlist
+cd /tmp/openlist && ./openlist start&
+/app/caddy/caddy run /app/Caddyfile &
+TASK_PID=$!
+
+# Wait for Open WebUI process
+wait $TASK_PID

sync_storage.py

@@ -0,0 +1,188 @@
+#!/usr/bin/env python3
+import os
+import shutil
+import json
+from pathlib import Path
+from huggingface_hub import HfApi, create_repo
+import tarfile
+import tempfile
+
+class HFStorageSync:
+    def __init__(self, repo_id, token=None, data_dir="/tmp/open-webui-data"):
+        self.repo_id = repo_id
+        self.data_dir = Path(data_dir)
+        self.token = token
+        
+        # Initialize API with token directly
+        self.api = HfApi(token=token) if token else HfApi()
+    
+    def ensure_repo_exists(self):
+        """Create repository if it doesn't exist"""
+        if not self.token:
+            print("No token provided, cannot create repository")
+            return False
+            
+        try:
+            # Check if repo exists
+            repo_info = self.api.repo_info(repo_id=self.repo_id, repo_type="dataset")
+            print(f"Repository {self.repo_id} exists")
+            return True
+        except Exception as e:
+            print(f"Repository {self.repo_id} not found, attempting to create...")
+            try:
+                create_repo(
+                    repo_id=self.repo_id,
+                    repo_type="dataset",
+                    token=self.token,
+                    private=True,  # Make it private by default
+                    exist_ok=True
+                )
+                print(f"Created repository {self.repo_id}")
+                
+                # Create initial README
+                readme_content = """# Open WebUI Storage
+
+This dataset stores persistent data for Open WebUI deployment.
+
+## Contents
+
+- `data.tar.gz`: Compressed archive containing all Open WebUI data including:
+  - User configurations
+  - Chat histories
+  - Uploaded files
+  - Database files
+
+This repository is automatically managed by the Open WebUI sync system.
+"""
+                
+                with tempfile.NamedTemporaryFile(mode='w', suffix='.md', delete=False) as tmp:
+                    tmp.write(readme_content)
+                    tmp.flush()
+                    
+                    self.api.upload_file(
+                        path_or_fileobj=tmp.name,
+                        path_in_repo="README.md",
+                        repo_id=self.repo_id,
+                        repo_type="dataset",
+                        commit_message="Initial repository setup",
+                        token=self.token
+                    )
+                    
+                    os.unlink(tmp.name)
+                
+                return True
+            except Exception as create_error:
+                print(f"Failed to create repository: {create_error}")
+                return False
+    
+    def download_data(self):
+        """Download and extract data from HF dataset repo"""
+        try:
+            print("Downloading data from Hugging Face...")
+            
+            # Ensure data directory exists and is writable
+            self.data_dir.mkdir(parents=True, exist_ok=True)
+            
+            # Test write permissions
+            test_file = self.data_dir / "test_write"
+            try:
+                test_file.touch()
+                test_file.unlink()
+                print(f"Data directory {self.data_dir} is writable")
+            except Exception as e:
+                print(f"Warning: Data directory may not be writable: {e}")
+                return
+            
+            if not self.token:
+                print("No HF_TOKEN provided, skipping download")
+                return
+            
+            # Ensure repository exists
+            if not self.ensure_repo_exists():
+                print("Could not access or create repository")
+                return
+            
+            # Try to download the data archive
+            try:
+                file_path = self.api.hf_hub_download(
+                    repo_id=self.repo_id,
+                    filename="data.tar.gz",
+                    repo_type="dataset",
+                    token=self.token
+                )
+                
+                with tarfile.open(file_path, 'r:gz') as tar:
+                    tar.extractall(self.data_dir)
+                
+                print(f"Data extracted to {self.data_dir}")
+                
+            except Exception as e:
+                print(f"No existing data found (this is normal for first run): {e}")
+            
+        except Exception as e:
+            print(f"Error during download: {e}")
+    
+    def upload_data(self):
+        """Compress and upload data to HF dataset repo"""
+        try:
+            if not self.token:
+                print("No HF_TOKEN provided, skipping upload")
+                return
+                
+            print("Uploading data to Hugging Face...")
+            
+            if not self.data_dir.exists() or not any(self.data_dir.iterdir()):
+                print("No data to upload")
+                return
+            
+            # Ensure repository exists
+            if not self.ensure_repo_exists():
+                print("Could not access or create repository")
+                return
+            
+            # Create temporary archive
+            with tempfile.NamedTemporaryFile(suffix='.tar.gz', delete=False) as tmp:
+                with tarfile.open(tmp.name, 'w:gz') as tar:
+                    for item in self.data_dir.iterdir():
+                        if item.name not in ["test_write", ".gitkeep"]:  # Skip test files
+                            tar.add(item, arcname=item.name)
+                
+                # Upload to HF
+                self.api.upload_file(
+                    path_or_fileobj=tmp.name,
+                    path_in_repo="data.tar.gz",
+                    repo_id=self.repo_id,
+                    repo_type="dataset",
+                    commit_message="Update Open WebUI data",
+                    token=self.token
+                )
+                
+                # Clean up
+                os.unlink(tmp.name)
+            
+            print("Data uploaded successfully")
+            
+        except Exception as e:
+            print(f"Error uploading data: {e}")
+
+def main():
+    import sys
+    
+    repo_id = os.getenv("HF_STORAGE_REPO", "nxdev-org/open-webui-storage")
+    token = os.getenv("HF_TOKEN")
+    data_dir = os.getenv("DATA_DIR", "/tmp/open-webui-data")
+    
+    sync = HFStorageSync(repo_id, token, data_dir)
+    
+    if len(sys.argv) > 1:
+        if sys.argv[1] == "download":
+            sync.download_data()
+        elif sys.argv[1] == "upload":
+            sync.upload_data()
+        else:
+            print("Usage: sync_storage.py [download|upload]")
+    else:
+        print("Usage: sync_storage.py [download|upload]")
+
+if __name__ == "__main__":
+    main()