in memory backup

backend/scripts/backup.py

@@ -5,42 +5,26 @@ import subprocess
 import datetime
 import sqlite3
 from pathlib import Path
-from huggingface_hub import HfApi, hf_hub_download, CommitOperationAdd
+from io import BytesIO
+from huggingface_hub import HfApi, HfFileSystem, hf_hub_download
+import tempfile
 
 # Set up path to include the application module
 SCRIPT_DIR = Path(__file__).parent.resolve()
 BACKEND_DIR = SCRIPT_DIR.parent
 sys.path.append(str(BACKEND_DIR))
 
-# Database paths
+# Database path (actual application database)
 DATA_DIR = os.environ.get("DATA_DIR", "/app/backend/data")
 DB_FILE_PATH = os.path.join(DATA_DIR, "webui.db")
 
-# Backup paths
-BACKUP_DIR = os.environ.get("BACKUP_DIR", "/tmp/open_webui/db_backup")
-TIMESTAMP_FILE_PATH = os.path.join(BACKUP_DIR, "last_backup_time.txt")
-DB_GPG_PATH = os.path.join(BACKUP_DIR, "webui.db.gpg")
-
-# Hugging Face repo paths
-REPO_TIMESTAMP_FILE = "db_backup/last_backup_time.txt"
+# Hugging Face repo paths (virtual paths in HF storage)
 REPO_DB_GPG_FILE = "db_backup/webui.db.gpg"
-
-
-def ensure_directories():
-    try:
-        os.makedirs(BACKUP_DIR, mode=0o755, exist_ok=True)
-        test_file = os.path.join(BACKUP_DIR, '.write_test')
-        with open(test_file, 'w') as f:
-            f.write('test')
-        os.remove(test_file)
-        print(f"Successfully verified write access to {BACKUP_DIR}")
-        return True
-    except Exception as e:
-        print(f"Error creating/verifying backup directory: {e}")
-        return False
+REPO_TIMESTAMP_FILE = "db_backup/last_backup_time.txt"
 
 
 def verify_database():
+    """Verify database integrity."""
     if not os.path.exists(DB_FILE_PATH):
         print(f"Database file not found at: {DB_FILE_PATH}")
         return False
@@ -56,46 +40,55 @@ def verify_database():
             if result.lower() == "ok" and len(tables) > 0:
                 print(f"Database verified: {len(tables)} tables found")
                 return True
-            else:
-                print("Database verification failed")
-                return False
+            print("Database verification failed")
+            return False
     except Exception as e:
         print(f"Database verification error: {e}")
         return False
 
 
-def encrypt_database(passphrase):
+def encrypt_database_to_memory(passphrase):
+    """Encrypt database directly to a memory buffer."""
     try:
-        os.makedirs('/root/.gnupg', mode=0o700, exist_ok=True)
-        
-        encrypt_cmd = [
-            "gpg",
-            "--batch",
-            "--yes",
-            "--passphrase", passphrase,
-            "--pinentry-mode", "loopback",
-            "-c",
-            "--cipher-algo", "AES256",
-            "-o", DB_GPG_PATH,
-            DB_FILE_PATH
-        ]
-        
-        result = subprocess.run(encrypt_cmd, capture_output=True, text=True)
-        
-        if result.returncode != 0:
-            print(f"GPG encryption failed: {result.stderr}")
-            return False
-        
-        if os.path.exists(DB_GPG_PATH):
-            print(f"Encryption successful. File size: {os.path.getsize(DB_GPG_PATH):,} bytes")
-            return True
-        return False
+        # Create a secure temporary directory for GPG
+        with tempfile.TemporaryDirectory(prefix='gpg_home_') as gpg_home:
+            os.chmod(gpg_home, 0o700)
+            
+            encrypt_cmd = [
+                "gpg",
+                "--batch",
+                "--yes",
+                "--homedir", gpg_home,
+                "--passphrase", passphrase,
+                "--pinentry-mode", "loopback",
+                "-c",
+                "--cipher-algo", "AES256",
+                "-o", "-",  # Output to stdout
+                DB_FILE_PATH
+            ]
+            
+            # Run GPG and capture output directly
+            result = subprocess.run(
+                encrypt_cmd,
+                capture_output=True,
+                check=True
+            )
+            
+            if result.returncode != 0:
+                print(f"GPG encryption failed: {result.stderr.decode()}")
+                return None
+                
+            return result.stdout
+    except subprocess.CalledProcessError as e:
+        print(f"GPG process error: {e.stderr.decode()}")
+        return None
     except Exception as e:
         print(f"Encryption error: {e}")
-        return False
+        return None
 
 
 def get_last_backup_time(repo_id, hf_token):
+    """Get timestamp of last backup from HuggingFace."""
     try:
         api = HfApi()
         files = api.list_repo_files(repo_id=repo_id, repo_type="space", token=hf_token)
@@ -116,33 +109,18 @@ def get_last_backup_time(repo_id, hf_token):
         return None
 
 
-def save_timestamp_locally():
-    try:
-        now = datetime.datetime.now(datetime.timezone.utc)
-        os.makedirs(os.path.dirname(TIMESTAMP_FILE_PATH), exist_ok=True)
-        
-        temp_path = f"{TIMESTAMP_FILE_PATH}.tmp"
-        with open(temp_path, "w", encoding="utf-8") as f:
-            f.write(now.isoformat())
-        os.replace(temp_path, TIMESTAMP_FILE_PATH)
-        return True
-    except Exception as e:
-        print(f"Error saving timestamp: {e}")
-        return False
-
-
 def backup_db():
+    """Main backup function using streaming approach."""
+    # Validate environment
     passphrase = os.environ.get("BACKUP_PASSPHRASE")
     hf_token = os.environ.get("HF_TOKEN")
     space_id = os.environ.get("SPACE_ID")
     
     if not all([passphrase, hf_token, space_id]):
-        print("Error: Missing required environment variables (BACKUP_PASSPHRASE, HF_TOKEN, SPACE_ID)")
+        print("Error: Missing required environment variables")
         return False
-    
-    if not ensure_directories():
-        return False
-    
+
+    # Check backup threshold
     threshold_minutes = int(os.environ.get("BACKUP_THRESHOLD_MINUTES", 120))
     if threshold_minutes > 0:
         last_backup_dt = get_last_backup_time(space_id, hf_token)
@@ -154,33 +132,43 @@ def backup_db():
             if elapsed.total_seconds() < threshold_minutes * 60:
                 print(f"Last backup was {elapsed.total_seconds()/60:.1f} min ago (threshold: {threshold_minutes})")
                 return True
-    
+
+    # Verify database before backup
     if not verify_database():
         return False
-    
-    if not encrypt_database(passphrase):
+
+    # Encrypt database to memory
+    print("Encrypting database...")
+    encrypted_data = encrypt_database_to_memory(passphrase)
+    if encrypted_data is None:
         return False
-    
-    if not save_timestamp_locally():
-        print("Warning: Failed to save timestamp")
-    
+    print(f"Database encrypted successfully: {len(encrypted_data)} bytes")
+
+    # Generate timestamp
+    timestamp = datetime.datetime.now(datetime.timezone.utc).isoformat()
+    timestamp_bytes = timestamp.encode('utf-8')
+
+    # Upload both files to HuggingFace
     try:
         api = HfApi()
-        operations = [
-            CommitOperationAdd(path_in_repo=REPO_DB_GPG_FILE, path_or_fileobj=DB_GPG_PATH),
-            CommitOperationAdd(path_in_repo=REPO_TIMESTAMP_FILE, path_or_fileobj=TIMESTAMP_FILE_PATH)
-        ]
-        api.create_commit(
+        api.upload_file(
+            path_or_fileobj=BytesIO(encrypted_data),
+            path_in_repo=REPO_DB_GPG_FILE,
+            repo_id=space_id,
+            repo_type="space",
+            token=hf_token
+        )
+        api.upload_file(
+            path_or_fileobj=BytesIO(timestamp_bytes),
+            path_in_repo=REPO_TIMESTAMP_FILE,
             repo_id=space_id,
             repo_type="space",
-            operations=operations,
-            commit_message="Update encrypted database backup + timestamp",
             token=hf_token
         )
-        print("Backup files uploaded successfully!")
+        print("Backup completed successfully!")
         return True
     except Exception as e:
-        print(f"Error uploading to HF: {e}")
+        print(f"Error uploading to HuggingFace: {e}")
         return False
 
 

backend/start.sh

@@ -5,7 +5,6 @@ SCRIPT_DIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")
 cd "$SCRIPT_DIR" || exit 1
 
 export DATA_DIR="/app/backend/data"
-export BACKUP_DIR="/tmp/open_webui/db_backup"
 export PYTHONPATH="/app/backend:${PYTHONPATH}"
 
 # Validate required environment variables