add dataset

Dockerfile

@@ -22,6 +22,10 @@ RUN chmod 666 /app/readeck/config.toml
 # can create and write files (like db.sqlite3) within this directory.
 RUN mkdir -p /app/readeck/data && chmod 777 /app/readeck/data
 
+# Copy sync scripts
+COPY sync_storage.py /app/sync_storage.py
+COPY start_with_sync.sh /start.sh
 
-
-CMD ["/bin/readeck","serve", "-config", "/app/readeck/config.toml"]
+# Start with sync
+ENTRYPOINT ["/start.sh"]
+# CMD ["/bin/readeck","serve", "-config", "/app/readeck/config.toml"]

start_with_sync.sh

@@ -0,0 +1,97 @@
+#!/bin/bash
+
+set -e
+
+# Use /tmp for all writable data
+export DATA_DIR="/app/readeck/data"
+export HF_STORAGE_REPO="${HF_STORAGE_REPO:-nxdev-org/open-webui-storage}"
+export SYNC_INTERVAL="${SYNC_INTERVAL:-300}"
+
+# Set all HuggingFace and cache directories to /tmp
+export HF_HOME="/tmp/hf_cache"
+export HUGGINGFACE_HUB_CACHE="/tmp/hf_cache"
+export TRANSFORMERS_CACHE="/tmp/hf_cache"
+export SENTENCE_TRANSFORMERS_HOME="/tmp/hf_cache"
+
+# Override Open WebUI environment variables
+export STATIC_DIR="/tmp/static"
+export UPLOAD_DIR="/tmp/uploads"
+
+echo "Starting Open WebUI with HF Dataset persistence..."
+echo "Data directory: $DATA_DIR"
+echo "HF Repository: $HF_STORAGE_REPO"
+echo "HF Cache: $HF_HOME"
+
+# Create all necessary directories
+mkdir -p "$DATA_DIR" "$HF_HOME" "$STATIC_DIR" "$UPLOAD_DIR"
+
+
+
+# Test write permissions
+if touch "$DATA_DIR/test" 2>/dev/null; then
+    rm "$DATA_DIR/test"
+    echo "Data directory is writable"
+else
+    echo "Warning: Data directory may not be writable"
+fi
+
+# Check if HF_TOKEN is set
+if [ -z "$HF_TOKEN" ]; then
+    echo "Warning: HF_TOKEN not set. Sync functionality will be limited."
+else
+    echo "HF_TOKEN is set, proceeding with sync..."
+fi
+
+# Download existing data on startup
+echo "Syncing data from Hugging Face..."
+python3 /app/sync_storage.py download
+
+# Function to handle graceful shutdown
+cleanup() {
+    echo "Shutting down gracefully..."
+    
+    # Upload final data state
+    if [ -n "$HF_TOKEN" ]; then
+        echo "Uploading final data state..."
+        python3 /app/sync_storage.py upload
+    fi
+    
+    # Kill background processes
+    kill $SYNC_PID 2>/dev/null || true
+    kill $WEBUI_PID 2>/dev/null || true
+    
+    exit 0
+}
+
+# Set up signal handlers
+trap cleanup SIGTERM SIGINT
+
+# Background sync function
+background_sync() {
+    if [ -n "$HF_TOKEN" ]; then
+        while true; do
+            sleep $SYNC_INTERVAL
+            echo "Periodic sync to Hugging Face..."
+            python3 /app/sync_storage.py upload
+        done
+    else
+        echo "Skipping background sync - no HF_TOKEN"
+        while true; do
+            sleep 3600
+        done
+    fi
+}
+
+# Start background sync
+background_sync &
+SYNC_PID=$!
+
+# Start Open WebUI
+echo "Starting Readeck..."
+
+# Start readeck in background
+/bin/readeck serve -config /app/readeck/config.toml &
+WEBUI_PID=$!
+
+# Wait for Open WebUI process
+wait $WEBUI_PID

sync_storage.py

@@ -0,0 +1,188 @@
+#!/usr/bin/env python3
+import os
+import shutil
+import json
+from pathlib import Path
+from huggingface_hub import HfApi, create_repo
+import tarfile
+import tempfile
+
+class HFStorageSync:
+    def __init__(self, repo_id, token=None, data_dir="/tmp/open-webui-data"):
+        self.repo_id = repo_id
+        self.data_dir = Path(data_dir)
+        self.token = token
+        
+        # Initialize API with token directly
+        self.api = HfApi(token=token) if token else HfApi()
+    
+    def ensure_repo_exists(self):
+        """Create repository if it doesn't exist"""
+        if not self.token:
+            print("No token provided, cannot create repository")
+            return False
+            
+        try:
+            # Check if repo exists
+            repo_info = self.api.repo_info(repo_id=self.repo_id, repo_type="dataset")
+            print(f"Repository {self.repo_id} exists")
+            return True
+        except Exception as e:
+            print(f"Repository {self.repo_id} not found, attempting to create...")
+            try:
+                create_repo(
+                    repo_id=self.repo_id,
+                    repo_type="dataset",
+                    token=self.token,
+                    private=True,  # Make it private by default
+                    exist_ok=True
+                )
+                print(f"Created repository {self.repo_id}")
+                
+                # Create initial README
+                readme_content = """# Open WebUI Storage
+
+This dataset stores persistent data for Open WebUI deployment.
+
+## Contents
+
+- `data.tar.gz`: Compressed archive containing all Open WebUI data including:
+  - User configurations
+  - Chat histories
+  - Uploaded files
+  - Database files
+
+This repository is automatically managed by the Open WebUI sync system.
+"""
+                
+                with tempfile.NamedTemporaryFile(mode='w', suffix='.md', delete=False) as tmp:
+                    tmp.write(readme_content)
+                    tmp.flush()
+                    
+                    self.api.upload_file(
+                        path_or_fileobj=tmp.name,
+                        path_in_repo="README.md",
+                        repo_id=self.repo_id,
+                        repo_type="dataset",
+                        commit_message="Initial repository setup",
+                        token=self.token
+                    )
+                    
+                    os.unlink(tmp.name)
+                
+                return True
+            except Exception as create_error:
+                print(f"Failed to create repository: {create_error}")
+                return False
+    
+    def download_data(self):
+        """Download and extract data from HF dataset repo"""
+        try:
+            print("Downloading data from Hugging Face...")
+            
+            # Ensure data directory exists and is writable
+            self.data_dir.mkdir(parents=True, exist_ok=True)
+            
+            # Test write permissions
+            test_file = self.data_dir / "test_write"
+            try:
+                test_file.touch()
+                test_file.unlink()
+                print(f"Data directory {self.data_dir} is writable")
+            except Exception as e:
+                print(f"Warning: Data directory may not be writable: {e}")
+                return
+            
+            if not self.token:
+                print("No HF_TOKEN provided, skipping download")
+                return
+            
+            # Ensure repository exists
+            if not self.ensure_repo_exists():
+                print("Could not access or create repository")
+                return
+            
+            # Try to download the data archive
+            try:
+                file_path = self.api.hf_hub_download(
+                    repo_id=self.repo_id,
+                    filename="data.tar.gz",
+                    repo_type="dataset",
+                    token=self.token
+                )
+                
+                with tarfile.open(file_path, 'r:gz') as tar:
+                    tar.extractall(self.data_dir)
+                
+                print(f"Data extracted to {self.data_dir}")
+                
+            except Exception as e:
+                print(f"No existing data found (this is normal for first run): {e}")
+            
+        except Exception as e:
+            print(f"Error during download: {e}")
+    
+    def upload_data(self):
+        """Compress and upload data to HF dataset repo"""
+        try:
+            if not self.token:
+                print("No HF_TOKEN provided, skipping upload")
+                return
+                
+            print("Uploading data to Hugging Face...")
+            
+            if not self.data_dir.exists() or not any(self.data_dir.iterdir()):
+                print("No data to upload")
+                return
+            
+            # Ensure repository exists
+            if not self.ensure_repo_exists():
+                print("Could not access or create repository")
+                return
+            
+            # Create temporary archive
+            with tempfile.NamedTemporaryFile(suffix='.tar.gz', delete=False) as tmp:
+                with tarfile.open(tmp.name, 'w:gz') as tar:
+                    for item in self.data_dir.iterdir():
+                        if item.name not in ["test_write", ".gitkeep"]:  # Skip test files
+                            tar.add(item, arcname=item.name)
+                
+                # Upload to HF
+                self.api.upload_file(
+                    path_or_fileobj=tmp.name,
+                    path_in_repo="data.tar.gz",
+                    repo_id=self.repo_id,
+                    repo_type="dataset",
+                    commit_message="Update Open WebUI data",
+                    token=self.token
+                )
+                
+                # Clean up
+                os.unlink(tmp.name)
+            
+            print("Data uploaded successfully")
+            
+        except Exception as e:
+            print(f"Error uploading data: {e}")
+
+def main():
+    import sys
+    
+    repo_id = os.getenv("HF_STORAGE_REPO", "nxdev-org/open-webui-storage")
+    token = os.getenv("HF_TOKEN")
+    data_dir = os.getenv("DATA_DIR", "/tmp/open-webui-data")
+    
+    sync = HFStorageSync(repo_id, token, data_dir)
+    
+    if len(sys.argv) > 1:
+        if sys.argv[1] == "download":
+            sync.download_data()
+        elif sys.argv[1] == "upload":
+            sync.upload_data()
+        else:
+            print("Usage: sync_storage.py [download|upload]")
+    else:
+        print("Usage: sync_storage.py [download|upload]")
+
+if __name__ == "__main__":
+    main()