more fixes

Dockerfile

@@ -133,7 +133,7 @@ RUN if [ "$USE_OLLAMA" = "true" ]; then \
 # install python dependencies
 COPY --chown=$UID:$GID ./backend/requirements.txt ./requirements.txt
 
-RUN pip3 install huggingface_hub
+RUN pip3 install huggingface_hub sqlite3
 RUN pip3 install uv && \
     if [ "$USE_CUDA" = "true" ]; then \
     # If you use CUDA the whisper and embedding model will be downloaded on first use

backend/requirements.txt

@@ -107,3 +107,4 @@ ldap3==2.9.1
 
 gnupg
 huggingface_hub
+sqlite3

backend/scripts/backup.py

@@ -3,12 +3,17 @@ import os
 import sys
 import subprocess
 import datetime
+import sqlite3
 from pathlib import Path
 from huggingface_hub import HfApi, hf_hub_download, CommitOperationAdd
 
+# Get the absolute paths for our script and base directories
+# We use absolute paths to ensure reliability regardless of the working directory
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
 BASE_DIR = os.path.dirname(SCRIPT_DIR)
 
+# Define all paths as absolute paths relative to our base directory
+# This prevents issues when the working directory changes during execution
 TIMESTAMP_FILE_PATH = os.path.join(BASE_DIR, "db_backup/last_backup_time.txt")
 DB_GPG_PATH = os.path.join(BASE_DIR, "db_backup/webui.db.gpg")
 DB_FILE_PATH = os.path.join(BASE_DIR, "data/webui.db")
@@ -16,10 +21,14 @@ DB_FILE_PATH = os.path.join(BASE_DIR, "data/webui.db")
 def ensure_directories():
     """
     Create all necessary directories for our backup operation.
-    Using absolute paths and proper permissions ensures consistent behavior.
+    This function runs before any file operations to ensure we have proper permissions
+    and directory structure in place.
+    
+    Returns:
+        bool: True if directories were created successfully, False otherwise
     """
     try:
-        # Create directories for both backup and database files
+        # Create both backup and database directories with appropriate permissions
         for directory in [os.path.dirname(DB_GPG_PATH), os.path.dirname(DB_FILE_PATH)]:
             os.makedirs(directory, mode=0o755, exist_ok=True)
         return True
@@ -30,7 +39,7 @@ def ensure_directories():
 def get_last_backup_time(repo_id, hf_token):
     """
     Retrieve the timestamp of the last backup from HuggingFace Space.
-    This helps us determine if we need to perform a new backup.
+    This helps us implement the backup threshold logic to prevent too frequent backups.
     
     Args:
         repo_id (str): The HuggingFace Space ID
@@ -41,7 +50,7 @@ def get_last_backup_time(repo_id, hf_token):
     """
     api = HfApi()
     try:
-        # First check if the file exists in the repository
+        # First check if the timestamp file exists in the repository
         files = api.list_repo_files(
             repo_id=repo_id,
             repo_type="space",
@@ -53,7 +62,7 @@ def get_last_backup_time(repo_id, hf_token):
             print(f"No timestamp file found in repository")
             return None
 
-        # Download and parse the timestamp file
+        # Download and parse the timestamp file if it exists
         temp_file = hf_hub_download(
             repo_id=repo_id,
             repo_type="space",
@@ -72,9 +81,13 @@ def get_last_backup_time(repo_id, hf_token):
 def save_timestamp_locally():
     """
     Save the current UTC time as our backup timestamp.
-    Creates the backup directory if it doesn't exist.
+    This timestamp is used to track when backups occur and implement threshold logic.
+    
+    Returns:
+        bool: True if timestamp was saved successfully, False otherwise
     """
     try:
+        # Use UTC time with timezone information for consistent timestamps
         now = datetime.datetime.now(datetime.timezone.utc)
         os.makedirs(os.path.dirname(TIMESTAMP_FILE_PATH), exist_ok=True)
         with open(TIMESTAMP_FILE_PATH, "w", encoding="utf-8") as f:
@@ -87,26 +100,65 @@ def save_timestamp_locally():
 def verify_database():
     """
     Verify the database exists and has basic integrity.
-    This prevents us from backing up corrupted data.
+    Uses Python's built-in sqlite3 module to perform comprehensive checks:
+    1. Verifies the file exists
+    2. Checks database integrity
+    3. Validates basic schema structure
+    
+    Returns:
+        bool: True if database is valid and healthy, False otherwise
     """
     if not os.path.exists(DB_FILE_PATH):
         print(f"Database file not found at: {DB_FILE_PATH}")
         return False
         
     try:
-        verify_cmd = ["sqlite3", DB_FILE_PATH, "PRAGMA integrity_check;"]
-        result = subprocess.run(verify_cmd, capture_output=True, text=True, check=True)
-        return "ok" in result.stdout.lower()
-    except subprocess.CalledProcessError as e:
+        # Attempt to connect to the database and perform verification
+        with sqlite3.connect(DB_FILE_PATH) as conn:
+            cursor = conn.cursor()
+            
+            # Run SQLite's built-in integrity check
+            cursor.execute("PRAGMA integrity_check;")
+            result = cursor.fetchone()[0]
+            
+            # Verify the database has a basic schema structure
+            cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
+            tables = cursor.fetchall()
+            
+            if result.lower() == "ok" and len(tables) > 0:
+                print("Database integrity verified successfully")
+                print(f"Found {len(tables)} tables in database")
+                return True
+            else:
+                print("Database integrity check failed")
+                if result.lower() != "ok":
+                    print(f"Integrity check result: {result}")
+                if len(tables) == 0:
+                    print("No tables found in database")
+                return False
+                
+    except sqlite3.Error as e:
         print(f"Database verification failed: {e}")
         return False
+    except Exception as e:
+        print(f"Unexpected error during database verification: {e}")
+        return False
 
 def backup_db():
     """
     Main function to handle the database backup process.
-    Includes verification, encryption, and upload to HuggingFace.
+    Orchestrates the entire backup operation including:
+    1. Environment validation
+    2. Directory creation
+    3. Threshold checking
+    4. Database verification
+    5. Encryption
+    6. Upload to HuggingFace
+    
+    Returns:
+        bool: True if backup completed successfully, False otherwise
     """
-    # Ensure we have all required environment variables
+    # Validate required environment variables
     passphrase = os.environ.get("BACKUP_PASSPHRASE")
     hf_token = os.environ.get("HF_TOKEN")
     space_id = os.environ.get("SPACE_ID")
@@ -115,14 +167,14 @@ def backup_db():
         print("Error: Missing required environment variables")
         return False
 
-    # Create necessary directories first
+    # Ensure our directory structure exists
     if not ensure_directories():
         return False
     
     # Get threshold from environment variable, defaulting to 2 hours
     threshold_minutes = int(os.environ.get("BACKUP_THRESHOLD_MINUTES", 120))
     
-    # Check if backup is needed based on threshold
+    # Check if we need to perform a backup based on the threshold
     if threshold_minutes > 0:
         last_backup_dt = get_last_backup_time(space_id, hf_token)
         if last_backup_dt is not None:
@@ -131,19 +183,19 @@ def backup_db():
                 last_backup_dt = last_backup_dt.replace(tzinfo=datetime.timezone.utc)
             elapsed = now - last_backup_dt
             if elapsed.total_seconds() < threshold_minutes * 60:
-                print(f"Last backup was only {elapsed.total_seconds()/3600:.2f} hours ago.")
-                print(f"Threshold is {threshold_minutes} minutes.")
-                print("Skipping backup to avoid rebuild loop.")
+                print(f"Last backup was only {elapsed.total_seconds()/3600:.2f} hours ago")
+                print(f"Threshold is {threshold_minutes} minutes")
+                print("Skipping backup to avoid rebuild loop")
                 return True
     else:
         print("Backup threshold check disabled for testing")
 
-    # Verify database before backing up
+    # Verify database integrity before proceeding
     if not verify_database():
         print("Database verification failed, aborting backup")
         return False
 
-    # Proceed with backup...
+    # Proceed with backup encryption
     print("Encrypting database with GPG...")
     encrypt_cmd = [
         "gpg",
@@ -164,16 +216,16 @@ def backup_db():
 
     print(f"Database encrypted successfully to {DB_GPG_PATH}")
 
-    # Update the timestamp file locally
+    # Save the backup timestamp
     if not save_timestamp_locally():
         return False
 
-    # Upload to Hugging Face Spaces
+    # Upload to HuggingFace Spaces
     print("Uploading to Hugging Face Spaces...")
     api = HfApi()
     
     try:
-        # Create operations list for both files
+        # Prepare file operations for both backup and timestamp
         operations = [
             CommitOperationAdd(
                 path_in_repo=os.path.relpath(DB_GPG_PATH, BASE_DIR),
@@ -185,6 +237,7 @@ def backup_db():
             )
         ]
 
+        # Commit both files to the repository
         api.create_commit(
             repo_id=space_id,
             repo_type="space",

backend/scripts/restore.py

@@ -3,14 +3,20 @@ import os
 import sys
 import subprocess
 import datetime
+import sqlite3
 from pathlib import Path
 from huggingface_hub import HfApi, hf_hub_download
 
-# Keep paths consistent with backup.py for better maintainability
-# Using relative paths allows the script to work regardless of where the repository is cloned
-TIMESTAMP_FILE_PATH = "db_backup/last_backup_time.txt"
-DB_GPG_PATH = "db_backup/webui.db.gpg"
-DB_FILE_PATH = "data/webui.db"
+# Just like in backup.py, we establish absolute paths based on the script's location
+# This ensures consistent path handling regardless of where the script is called from
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+BASE_DIR = os.path.dirname(SCRIPT_DIR)
+
+# Define all paths as absolute paths relative to our base directory
+# This maintains consistency with backup.py and prevents path-related issues
+TIMESTAMP_FILE_PATH = os.path.join(BASE_DIR, "db_backup/last_backup_time.txt")
+DB_GPG_PATH = os.path.join(BASE_DIR, "db_backup/webui.db.gpg")
+DB_FILE_PATH = os.path.join(BASE_DIR, "data/webui.db")
 
 def check_requirements():
     """
@@ -21,6 +27,7 @@ def check_requirements():
         bool: True if GPG is available, False otherwise
     """
     try:
+        # Test GPG availability by checking its version
         subprocess.run(["gpg", "--version"], check=True, capture_output=True)
         return True
     except (subprocess.CalledProcessError, FileNotFoundError):
@@ -29,8 +36,8 @@ def check_requirements():
 
 def validate_secrets():
     """
-    Check that all required environment variables are set.
-    These variables are essential for accessing the backup and decrypting it.
+    Ensure all required environment variables are set.
+    These variables are essential for accessing and decrypting the backup.
     
     Returns:
         bool: True if all required variables are set, False otherwise
@@ -43,41 +50,17 @@ def validate_secrets():
         return False
     return True
 
-def validate_repo_access(repo_id, token):
-    """
-    Verify that we can access the HuggingFace repository with the provided credentials.
-    This early check helps prevent cryptic errors later in the process.
-    
-    Args:
-        repo_id (str): The HuggingFace Space ID
-        token (str): The HuggingFace API token
-    
-    Returns:
-        bool: True if repository access is confirmed, False otherwise
-    """
-    api = HfApi()
-    try:
-        api.list_repo_files(
-            repo_id=repo_id,
-            repo_type="space",
-            token=token
-        )
-        return True
-    except Exception as e:
-        print(f"Error validating repository access: {e}")
-        print(f"Please verify your SPACE_ID ({repo_id}) and HF_TOKEN are correct")
-        return False
-
 def ensure_directories():
     """
-    Create the necessary directory structure for database and backup files.
-    This ensures we have proper permissions and all required directories exist.
+    Create necessary directories for database and backup files.
+    This ensures we have the proper directory structure before any operations.
     
     Returns:
         bool: True if directories are created successfully, False otherwise
     """
     try:
-        for directory in ["data", "db_backup"]:
+        # Create both backup and database directories with appropriate permissions
+        for directory in [os.path.dirname(DB_FILE_PATH), os.path.dirname(DB_GPG_PATH)]:
             os.makedirs(directory, mode=0o755, exist_ok=True)
         return True
     except Exception as e:
@@ -87,7 +70,6 @@ def ensure_directories():
 def get_latest_backup_info(repo_id, hf_token):
     """
     Check for and retrieve information about the latest backup from HuggingFace.
-    This function both verifies backup existence and gets its timestamp.
     
     Args:
         repo_id (str): The HuggingFace Space ID
@@ -98,14 +80,16 @@ def get_latest_backup_info(repo_id, hf_token):
     """
     api = HfApi()
     try:
-        # First check if the backup file exists in the repository
+        # Check if backup file exists in the repository
         files = api.list_repo_files(
             repo_id=repo_id,
             repo_type="space",
             token=hf_token
         )
         
-        backup_exists = DB_GPG_PATH in files
+        # Look for backup file using relative path
+        relative_backup_path = os.path.relpath(DB_GPG_PATH, BASE_DIR)
+        backup_exists = relative_backup_path in files
         
         if not backup_exists:
             print("No backup file found in the repository")
@@ -113,10 +97,11 @@ def get_latest_backup_info(repo_id, hf_token):
             
         # If backup exists, try to get its timestamp
         try:
+            relative_timestamp_path = os.path.relpath(TIMESTAMP_FILE_PATH, BASE_DIR)
             timestamp_file = hf_hub_download(
                 repo_id=repo_id,
                 repo_type="space",
-                filename=TIMESTAMP_FILE_PATH,
+                filename=relative_timestamp_path,
                 token=hf_token
             )
             
@@ -142,7 +127,6 @@ def get_latest_backup_info(repo_id, hf_token):
 def download_backup(repo_id, hf_token):
     """
     Download the encrypted database backup from HuggingFace.
-    Handles the safe download and placement of the backup file.
     
     Args:
         repo_id (str): The HuggingFace Space ID
@@ -153,10 +137,11 @@ def download_backup(repo_id, hf_token):
     """
     try:
         print("Downloading encrypted database backup...")
+        relative_backup_path = os.path.relpath(DB_GPG_PATH, BASE_DIR)
         temp_file = hf_hub_download(
             repo_id=repo_id,
             repo_type="space",
-            filename=DB_GPG_PATH,
+            filename=relative_backup_path,
             token=hf_token
         )
         
@@ -208,6 +193,7 @@ def decrypt_database(passphrase):
 def verify_database():
     """
     Verify the integrity of the restored database using SQLite's built-in checks.
+    This function uses Python's sqlite3 module instead of the command-line tool.
     
     Returns:
         bool: True if database is valid or doesn't exist, False if corrupted
@@ -217,24 +203,45 @@ def verify_database():
         
     try:
         print("Verifying database integrity...")
-        verify_cmd = ["sqlite3", DB_FILE_PATH, "PRAGMA integrity_check;"]
-        result = subprocess.run(verify_cmd, capture_output=True, text=True, check=True)
-        
-        if "ok" in result.stdout.lower():
-            print("Database integrity verified")
-            return True
-        else:
-            print("Database integrity check failed")
-            return False
+        with sqlite3.connect(DB_FILE_PATH) as conn:
+            cursor = conn.cursor()
             
-    except subprocess.CalledProcessError as e:
+            # Run integrity check
+            cursor.execute("PRAGMA integrity_check;")
+            result = cursor.fetchone()[0]
+            
+            # Verify basic schema structure
+            cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
+            tables = cursor.fetchall()
+            
+            if result.lower() == "ok" and len(tables) > 0:
+                print("Database integrity verified successfully")
+                print(f"Found {len(tables)} tables in database")
+                return True
+            else:
+                print("Database integrity check failed")
+                if result.lower() != "ok":
+                    print(f"Integrity check result: {result}")
+                if len(tables) == 0:
+                    print("No tables found in database")
+                return False
+            
+    except sqlite3.Error as e:
         print(f"Database verification failed: {e}")
         return False
+    except Exception as e:
+        print(f"Unexpected error during database verification: {e}")
+        return False
 
 def restore_db():
     """
-    Main function that orchestrates the database restoration process.
-    Performs all necessary checks and handles the complete restore operation.
+    Main function to handle the database restoration process.
+    This function orchestrates the complete restore operation, including:
+    1. Environment validation
+    2. Directory creation
+    3. Backup download
+    4. Decryption
+    5. Database verification
     
     Returns:
         bool: True if restore is successful or no backup needed, False on error
@@ -252,10 +259,6 @@ def restore_db():
     hf_token = os.environ["HF_TOKEN"]
     space_id = os.environ["SPACE_ID"]
     
-    # Validate repository access first
-    if not validate_repo_access(space_id, hf_token):
-        return False
-    
     # Check if there's a backup in the repository
     backup_exists, timestamp = get_latest_backup_info(space_id, hf_token)