more fixes

backend/scripts/backup.py

@@ -8,38 +8,55 @@ from pathlib import Path
 from huggingface_hub import HfApi, hf_hub_download, CommitOperationAdd
 
 # Get the absolute paths for our script and base directories
-# We use absolute paths to ensure reliability regardless of the working directory
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
 BASE_DIR = os.path.dirname(SCRIPT_DIR)
 
 # Define all paths as absolute paths relative to our base directory
-# This prevents issues when the working directory changes during execution
 TIMESTAMP_FILE_PATH = os.path.join(BASE_DIR, "db_backup/last_backup_time.txt")
 DB_GPG_PATH = os.path.join(BASE_DIR, "db_backup/webui.db.gpg")
 DB_FILE_PATH = os.path.join(BASE_DIR, "data/webui.db")
 
 def ensure_directories():
     """
-    Create all necessary directories for our backup operation.
-    This function runs before any file operations to ensure we have proper permissions
-    and directory structure in place.
+    Create and verify all necessary directories for our backup operation.
+    Includes thorough permission checking and write verification.
     
     Returns:
-        bool: True if directories were created successfully, False otherwise
+        bool: True if directories were created and verified successfully
     """
     try:
-        # Create both backup and database directories with appropriate permissions
         for directory in [os.path.dirname(DB_GPG_PATH), os.path.dirname(DB_FILE_PATH)]:
+            # Create directory with full permissions for the current user
             os.makedirs(directory, mode=0o755, exist_ok=True)
+            
+            # Verify directory exists and get its current permissions
+            dir_stat = os.stat(directory)
+            print(f"Directory {directory} created with permissions: {oct(dir_stat.st_mode)[-3:]}")
+            
+            # Verify we can write to the directory with a test file
+            test_file = os.path.join(directory, '.write_test')
+            try:
+                with open(test_file, 'w') as f:
+                    f.write('test')
+                os.remove(test_file)
+                print(f"Successfully verified write access to {directory}")
+            except Exception as e:
+                print(f"Directory {directory} is not writable: {e}")
+                print(f"Current process user ID: {os.getuid()}")
+                print(f"Directory owner ID: {dir_stat.st_uid}")
+                return False
+                
         return True
     except Exception as e:
-        print(f"Error creating directories: {e}")
+        print(f"Error in directory creation/verification: {e}")
+        print(f"Current process user ID: {os.getuid()}")
+        print(f"Current process group ID: {os.getgid()}")
+        print(f"Current working directory: {os.getcwd()}")
         return False
 
 def get_last_backup_time(repo_id, hf_token):
     """
     Retrieve the timestamp of the last backup from HuggingFace Space.
-    This helps us implement the backup threshold logic to prevent too frequent backups.
     
     Args:
         repo_id (str): The HuggingFace Space ID
@@ -50,7 +67,7 @@ def get_last_backup_time(repo_id, hf_token):
     """
     api = HfApi()
     try:
-        # First check if the timestamp file exists in the repository
+        # Check if the timestamp file exists in the repository
         files = api.list_repo_files(
             repo_id=repo_id,
             repo_type="space",
@@ -62,7 +79,7 @@ def get_last_backup_time(repo_id, hf_token):
             print(f"No timestamp file found in repository")
             return None
 
-        # Download and parse the timestamp file if it exists
+        # Download and parse the timestamp file
         temp_file = hf_hub_download(
             repo_id=repo_id,
             repo_type="space",
@@ -81,17 +98,24 @@ def get_last_backup_time(repo_id, hf_token):
 def save_timestamp_locally():
     """
     Save the current UTC time as our backup timestamp.
-    This timestamp is used to track when backups occur and implement threshold logic.
+    Includes error handling and directory creation.
     
     Returns:
-        bool: True if timestamp was saved successfully, False otherwise
+        bool: True if timestamp was saved successfully
     """
     try:
-        # Use UTC time with timezone information for consistent timestamps
+        # Use UTC time with timezone information
         now = datetime.datetime.now(datetime.timezone.utc)
-        os.makedirs(os.path.dirname(TIMESTAMP_FILE_PATH), exist_ok=True)
-        with open(TIMESTAMP_FILE_PATH, "w", encoding="utf-8") as f:
+        
+        # Ensure the directory exists
+        os.makedirs(os.path.dirname(TIMESTAMP_FILE_PATH), mode=0o755, exist_ok=True)
+        
+        # Write the timestamp atomically using a temporary file
+        temp_path = f"{TIMESTAMP_FILE_PATH}.tmp"
+        with open(temp_path, "w", encoding="utf-8") as f:
             f.write(now.isoformat())
+        os.replace(temp_path, TIMESTAMP_FILE_PATH)
+        
         return True
     except Exception as e:
         print(f"Error saving timestamp: {e}")
@@ -99,29 +123,31 @@ def save_timestamp_locally():
 
 def verify_database():
     """
-    Verify the database exists and has basic integrity.
-    Uses Python's built-in sqlite3 module to perform comprehensive checks:
-    1. Verifies the file exists
-    2. Checks database integrity
-    3. Validates basic schema structure
+    Comprehensive verification of the database's existence and integrity.
+    Uses SQLite's built-in integrity check and schema validation.
     
     Returns:
-        bool: True if database is valid and healthy, False otherwise
+        bool: True if database is valid and healthy
     """
     if not os.path.exists(DB_FILE_PATH):
         print(f"Database file not found at: {DB_FILE_PATH}")
         return False
         
     try:
-        # Attempt to connect to the database and perform verification
+        # Get file information for debugging
+        file_stat = os.stat(DB_FILE_PATH)
+        print(f"Database file size: {file_stat.st_size:,} bytes")
+        print(f"Database file permissions: {oct(file_stat.st_mode)[-3:]}")
+        
+        # Verify database integrity
         with sqlite3.connect(DB_FILE_PATH) as conn:
             cursor = conn.cursor()
             
-            # Run SQLite's built-in integrity check
+            # Run comprehensive integrity check
             cursor.execute("PRAGMA integrity_check;")
             result = cursor.fetchone()[0]
             
-            # Verify the database has a basic schema structure
+            # Verify schema structure
             cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
             tables = cursor.fetchall()
             
@@ -138,27 +164,103 @@ def verify_database():
                 return False
                 
     except sqlite3.Error as e:
-        print(f"Database verification failed: {e}")
+        print(f"SQLite error during verification: {e}")
+        return False
+    except Exception as e:
+        print(f"Unexpected error during verification: {e}")
+        print(f"Error type: {type(e).__name__}")
         return False
+
+def encrypt_database(passphrase):
+    """
+    Encrypt the database file using GPG with comprehensive error checking.
+    
+    Args:
+        passphrase (str): The encryption passphrase
+    
+    Returns:
+        bool: True if encryption was successful
+    """
+    try:
+        print("\nPreparing for database encryption...")
+        
+        # Verify source file
+        if not os.path.exists(DB_FILE_PATH):
+            print(f"Source database file not found: {DB_FILE_PATH}")
+            return False
+            
+        source_size = os.path.getsize(DB_FILE_PATH)
+        print(f"Source file exists and is {source_size:,} bytes")
+        
+        # Verify target directory
+        target_dir = os.path.dirname(DB_GPG_PATH)
+        if not os.path.exists(target_dir):
+            print(f"Target directory doesn't exist: {target_dir}")
+            return False
+            
+        print(f"Target directory exists and {'is' if os.access(target_dir, os.W_OK) else 'is not'} writable")
+        
+        # Remove existing backup file if it exists
+        if os.path.exists(DB_GPG_PATH):
+            try:
+                os.remove(DB_GPG_PATH)
+                print("Removed existing backup file")
+            except Exception as e:
+                print(f"Warning: Could not remove existing backup: {e}")
+                return False
+        
+        # Prepare GPG command
+        encrypt_cmd = [
+            "gpg",
+            "--batch",
+            "--yes",
+            "--passphrase", passphrase,
+            "-c",
+            "--cipher-algo", "AES256",
+            "-o", DB_GPG_PATH,
+            DB_FILE_PATH
+        ]
+        
+        # Run GPG with output capturing
+        print("\nRunning GPG encryption...")
+        result = subprocess.run(
+            encrypt_cmd,
+            capture_output=True,
+            text=True,
+            check=False
+        )
+        
+        if result.returncode != 0:
+            print(f"GPG encryption failed with code {result.returncode}")
+            if result.stdout:
+                print(f"GPG stdout: {result.stdout}")
+            if result.stderr:
+                print(f"GPG stderr: {result.stderr}")
+            return False
+        
+        # Verify the encrypted file was created
+        if not os.path.exists(DB_GPG_PATH):
+            print("GPG reported success but encrypted file not found")
+            return False
+            
+        encrypted_size = os.path.getsize(DB_GPG_PATH)
+        print(f"Encryption successful. Encrypted file size: {encrypted_size:,} bytes")
+        return True
+        
     except Exception as e:
-        print(f"Unexpected error during database verification: {e}")
+        print(f"Encryption failed with exception: {e}")
+        print(f"Exception type: {type(e).__name__}")
         return False
 
 def backup_db():
     """
-    Main function to handle the database backup process.
-    Orchestrates the entire backup operation including:
-    1. Environment validation
-    2. Directory creation
-    3. Threshold checking
-    4. Database verification
-    5. Encryption
-    6. Upload to HuggingFace
+    Main function to handle the complete database backup process.
+    Includes environment validation, threshold checking, encryption, and upload.
     
     Returns:
-        bool: True if backup completed successfully, False otherwise
+        bool: True if backup completed successfully
     """
-    # Validate required environment variables
+    # Validate environment variables
     passphrase = os.environ.get("BACKUP_PASSPHRASE")
     hf_token = os.environ.get("HF_TOKEN")
     space_id = os.environ.get("SPACE_ID")
@@ -167,14 +269,15 @@ def backup_db():
         print("Error: Missing required environment variables")
         return False
 
-    # Ensure our directory structure exists
+    # Ensure directory structure exists
     if not ensure_directories():
+        print("Failed to create or verify directories")
         return False
     
-    # Get threshold from environment variable, defaulting to 2 hours
+    # Get backup threshold setting
     threshold_minutes = int(os.environ.get("BACKUP_THRESHOLD_MINUTES", 120))
     
-    # Check if we need to perform a backup based on the threshold
+    # Check if backup is needed based on threshold
     if threshold_minutes > 0:
         last_backup_dt = get_last_backup_time(space_id, hf_token)
         if last_backup_dt is not None:
@@ -190,42 +293,27 @@ def backup_db():
     else:
         print("Backup threshold check disabled for testing")
 
-    # Verify database integrity before proceeding
+    # Verify database integrity
     if not verify_database():
         print("Database verification failed, aborting backup")
         return False
 
-    # Proceed with backup encryption
-    print("Encrypting database with GPG...")
-    encrypt_cmd = [
-        "gpg",
-        "--batch",
-        "--yes",
-        "--passphrase", passphrase,
-        "-c",
-        "--cipher-algo", "AES256",
-        "-o", DB_GPG_PATH,
-        DB_FILE_PATH
-    ]
-    
-    try:
-        subprocess.run(encrypt_cmd, check=True)
-    except subprocess.CalledProcessError as e:
-        print(f"Encryption failed: {e}")
+    # Encrypt the database
+    if not encrypt_database(passphrase):
+        print("Database encryption failed, aborting backup")
         return False
 
-    print(f"Database encrypted successfully to {DB_GPG_PATH}")
-
     # Save the backup timestamp
     if not save_timestamp_locally():
+        print("Failed to save timestamp")
         return False
 
     # Upload to HuggingFace Spaces
-    print("Uploading to Hugging Face Spaces...")
+    print("\nUploading to Hugging Face Spaces...")
     api = HfApi()
     
     try:
-        # Prepare file operations for both backup and timestamp
+        # Prepare file operations
         operations = [
             CommitOperationAdd(
                 path_in_repo=os.path.relpath(DB_GPG_PATH, BASE_DIR),
@@ -237,7 +325,7 @@ def backup_db():
             )
         ]
 
-        # Commit both files to the repository
+        # Create commit with both files
         api.create_commit(
             repo_id=space_id,
             repo_type="space",
@@ -245,7 +333,7 @@ def backup_db():
             commit_message="Update encrypted database backup + timestamp",
             token=hf_token
         )
-        print("DB backup + timestamp uploaded successfully!")
+        print("Backup files uploaded successfully!")
         return True
         
     except Exception as e: