more fixes

backend/scripts/backup.py

@@ -1,110 +1,204 @@
 #!/usr/bin/env python3
 import os
+import sys
 import subprocess
 import datetime
+from pathlib import Path
 from huggingface_hub import HfApi, hf_hub_download, CommitOperationAdd
 
-TIMESTAMP_FILE_PATH = "db_backup/last_backup_time.txt"
-DB_GPG_PATH = "db_backup/webui.db.gpg"
-DB_FILE_PATH = "data/webui.db"
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+BASE_DIR = os.path.dirname(SCRIPT_DIR)
+
+TIMESTAMP_FILE_PATH = os.path.join(BASE_DIR, "db_backup/last_backup_time.txt")
+DB_GPG_PATH = os.path.join(BASE_DIR, "db_backup/webui.db.gpg")
+DB_FILE_PATH = os.path.join(BASE_DIR, "data/webui.db")
+
+def ensure_directories():
+    """
+    Create all necessary directories for our backup operation.
+    Using absolute paths and proper permissions ensures consistent behavior.
+    """
+    try:
+        # Create directories for both backup and database files
+        for directory in [os.path.dirname(DB_GPG_PATH), os.path.dirname(DB_FILE_PATH)]:
+            os.makedirs(directory, mode=0o755, exist_ok=True)
+        return True
+    except Exception as e:
+        print(f"Error creating directories: {e}")
+        return False
 
 def get_last_backup_time(repo_id, hf_token):
     """
-    Attempt to download and parse the last_backup_time.txt file
-    from the HF Space. If it doesn't exist, return None.
+    Retrieve the timestamp of the last backup from HuggingFace Space.
+    This helps us determine if we need to perform a new backup.
+    
+    Args:
+        repo_id (str): The HuggingFace Space ID
+        hf_token (str): The HuggingFace API token
+    
+    Returns:
+        datetime or None: The timestamp of the last backup, or None if not found
     """
     api = HfApi()
     try:
-        temp_file = hf_hub_download(
+        # First check if the file exists in the repository
+        files = api.list_repo_files(
             repo_id=repo_id,
             repo_type="space",
-            filename=TIMESTAMP_FILE_PATH,
             token=hf_token
         )
-    except Exception as e:
-        print(f"Could not download {TIMESTAMP_FILE_PATH}: {e}")
-        return None
+        
+        relative_timestamp_path = os.path.relpath(TIMESTAMP_FILE_PATH, BASE_DIR)
+        if relative_timestamp_path not in files:
+            print(f"No timestamp file found in repository")
+            return None
 
-    try:
+        # Download and parse the timestamp file
+        temp_file = hf_hub_download(
+            repo_id=repo_id,
+            repo_type="space",
+            filename=relative_timestamp_path,
+            token=hf_token
+        )
+        
         with open(temp_file, "r", encoding="utf-8") as f:
             timestamp_str = f.read().strip()
-        last_backup_dt = datetime.datetime.fromisoformat(timestamp_str)
-        return last_backup_dt
+        return datetime.datetime.fromisoformat(timestamp_str)
+        
     except Exception as e:
-        print(f"Error parsing timestamp from {TIMESTAMP_FILE_PATH}: {e}")
+        print(f"Error getting last backup time: {e}")
         return None
 
 def save_timestamp_locally():
-    """Save the current UTC time to db_backup/last_backup_time.txt (locally)."""
-    now = datetime.datetime.utcnow()
-    os.makedirs("db_backup", exist_ok=True)
-    with open(TIMESTAMP_FILE_PATH, "w", encoding="utf-8") as f:
-        f.write(now.isoformat())
+    """
+    Save the current UTC time as our backup timestamp.
+    Creates the backup directory if it doesn't exist.
+    """
+    try:
+        now = datetime.datetime.now(datetime.timezone.utc)
+        os.makedirs(os.path.dirname(TIMESTAMP_FILE_PATH), exist_ok=True)
+        with open(TIMESTAMP_FILE_PATH, "w", encoding="utf-8") as f:
+            f.write(now.isoformat())
+        return True
+    except Exception as e:
+        print(f"Error saving timestamp: {e}")
+        return False
+
+def verify_database():
+    """
+    Verify the database exists and has basic integrity.
+    This prevents us from backing up corrupted data.
+    """
+    if not os.path.exists(DB_FILE_PATH):
+        print(f"Database file not found at: {DB_FILE_PATH}")
+        return False
+        
+    try:
+        verify_cmd = ["sqlite3", DB_FILE_PATH, "PRAGMA integrity_check;"]
+        result = subprocess.run(verify_cmd, capture_output=True, text=True, check=True)
+        return "ok" in result.stdout.lower()
+    except subprocess.CalledProcessError as e:
+        print(f"Database verification failed: {e}")
+        return False
 
 def backup_db():
-    # Get environment variables
+    """
+    Main function to handle the database backup process.
+    Includes verification, encryption, and upload to HuggingFace.
+    """
+    # Ensure we have all required environment variables
     passphrase = os.environ.get("BACKUP_PASSPHRASE")
     hf_token = os.environ.get("HF_TOKEN")
     space_id = os.environ.get("SPACE_ID")
     
-    # Get threshold from environment variable, defaulting to 2 hours if not set
-    # For testing, set BACKUP_THRESHOLD_MINUTES=0 to disable the threshold check
+    if not all([passphrase, hf_token, space_id]):
+        print("Error: Missing required environment variables")
+        return False
+
+    # Create necessary directories first
+    if not ensure_directories():
+        return False
+    
+    # Get threshold from environment variable, defaulting to 2 hours
     threshold_minutes = int(os.environ.get("BACKUP_THRESHOLD_MINUTES", 120))
     
-    if not passphrase:
-        raise ValueError("BACKUP_PASSPHRASE is not set.")
-    if not hf_token:
-        raise ValueError("HF_TOKEN is not set.")
-    if not space_id:
-        raise ValueError("SPACE_ID is not set (or define repo_id manually).")
-
     # Check if backup is needed based on threshold
-    if threshold_minutes > 0:  # Only check if threshold is positive
+    if threshold_minutes > 0:
         last_backup_dt = get_last_backup_time(space_id, hf_token)
         if last_backup_dt is not None:
-            now = datetime.datetime.utcnow()
+            now = datetime.datetime.now(datetime.timezone.utc)
+            if not last_backup_dt.tzinfo:
+                last_backup_dt = last_backup_dt.replace(tzinfo=datetime.timezone.utc)
             elapsed = now - last_backup_dt
             if elapsed.total_seconds() < threshold_minutes * 60:
                 print(f"Last backup was only {elapsed.total_seconds()/3600:.2f} hours ago.")
                 print(f"Threshold is {threshold_minutes} minutes.")
                 print("Skipping backup to avoid rebuild loop.")
-                return
+                return True
     else:
         print("Backup threshold check disabled for testing")
 
+    # Verify database before backing up
+    if not verify_database():
+        print("Database verification failed, aborting backup")
+        return False
+
     # Proceed with backup...
     print("Encrypting database with GPG...")
     encrypt_cmd = [
-        "gpg", "--batch", "--yes", "--passphrase", passphrase,
-        "-c", "--cipher-algo", "AES256",
+        "gpg",
+        "--batch",
+        "--yes",
+        "--passphrase", passphrase,
+        "-c",
+        "--cipher-algo", "AES256",
         "-o", DB_GPG_PATH,
         DB_FILE_PATH
     ]
-    subprocess.run(encrypt_cmd, check=True)
+    
+    try:
+        subprocess.run(encrypt_cmd, check=True)
+    except subprocess.CalledProcessError as e:
+        print(f"Encryption failed: {e}")
+        return False
+
     print(f"Database encrypted successfully to {DB_GPG_PATH}")
 
     # Update the timestamp file locally
-    save_timestamp_locally()
+    if not save_timestamp_locally():
+        return False
 
     # Upload to Hugging Face Spaces
     print("Uploading to Hugging Face Spaces...")
     api = HfApi()
-    repo_id = space_id
-
-    # Create operations list for both files
-    operations = [
-        CommitOperationAdd(path_in_repo=DB_GPG_PATH, path_or_fileobj=DB_GPG_PATH),
-        CommitOperationAdd(path_in_repo=TIMESTAMP_FILE_PATH, path_or_fileobj=TIMESTAMP_FILE_PATH)
-    ]
+    
+    try:
+        # Create operations list for both files
+        operations = [
+            CommitOperationAdd(
+                path_in_repo=os.path.relpath(DB_GPG_PATH, BASE_DIR),
+                path_or_fileobj=DB_GPG_PATH
+            ),
+            CommitOperationAdd(
+                path_in_repo=os.path.relpath(TIMESTAMP_FILE_PATH, BASE_DIR),
+                path_or_fileobj=TIMESTAMP_FILE_PATH
+            )
+        ]
 
-    api.create_commit(
-        repo_id=repo_id,
-        repo_type="space",
-        operations=operations,
-        commit_message="Update encrypted database backup + timestamp",
-        token=hf_token
-    )
-    print("DB backup + timestamp uploaded successfully!")
+        api.create_commit(
+            repo_id=space_id,
+            repo_type="space",
+            operations=operations,
+            commit_message="Update encrypted database backup + timestamp",
+            token=hf_token
+        )
+        print("DB backup + timestamp uploaded successfully!")
+        return True
+        
+    except Exception as e:
+        print(f"Error uploading to HuggingFace: {e}")
+        return False
 
 if __name__ == "__main__":
-    backup_db()
+    success = backup_db()
+    sys.exit(0 if success else 1)