# backup.py - SAOS Cognitive Backup & Git-Archiver Daemon import os import sys import time import gzip import shutil import logging import subprocess from datetime import datetime # Configure clean terminal logging logging.basicConfig( level=logging.INFO, format="%(asctime)s | %(levelname)-8s | %(name)s - %(message)s", handlers=[logging.StreamHandler(sys.stdout)], ) logger = logging.getLogger("saos.backup") # ── Cloud Credentials & Environment variables ────────────────────────────────── R2_ACCESS_KEY = os.getenv("R2_ACCESS_KEY", "") R2_SECRET_KEY = os.getenv("R2_SECRET_KEY", "") R2_ACCOUNT_ID = os.getenv("R2_ACCOUNT_ID", "") R2_BUCKET = os.getenv("R2_BUCKET", "saos-backups") GITHUB_PAT = os.getenv("GITHUB_PAT", "") GITHUB_USER = os.getenv("GITHUB_USER", "") ACTIVE_REPO_NAME = os.getenv("GITHUB_REPO", "saos-backups-part-1") REDIS_DATA_DIR = "/data" REDIS_DUMP_PATH = os.path.join(REDIS_DATA_DIR, "dump.rdb") COMPRESSED_DUMP_PATH = os.path.join(REDIS_DATA_DIR, "dump.rdb.gz") # ── Cloudflare R2 Sync Helper ───────────────────────────────────────────────── def compress_dump() -> bool: """Compresses dump.rdb to gzip with maximum compression (level 9).""" if not os.path.exists(REDIS_DUMP_PATH): logger.info("No active Redis dump.rdb file found to compress.") return False try: logger.info(f"Compressing {REDIS_DUMP_PATH} to {COMPRESSED_DUMP_PATH}...") with open(REDIS_DUMP_PATH, 'rb') as f_in: with gzip.open(COMPRESSED_DUMP_PATH, 'wb', compresslevel=9) as f_out: shutil.copyfileobj(f_in, f_out) logger.info("Successfully compressed Redis database dump!") return True except Exception as e: logger.error(f"Failed to compress Redis database dump: {e}") return False # ── Cloudflare R2 Sync Helper ───────────────────────────────────────────────── def upload_to_r2() -> bool: """Uploads pre-compressed dump.rdb.gz to Cloudflare R2 S3 bucket.""" if not (R2_ACCESS_KEY and R2_SECRET_KEY and R2_ACCOUNT_ID): logger.warning("Cloudflare R2 credentials missing. Skipping R2 swap upload.") return False if not os.path.exists(COMPRESSED_DUMP_PATH): logger.info("No compressed dump.rdb.gz found to upload to R2.") return False try: # Upload to S3-compatible R2 endpoint import boto3 from botocore.client import Config r2_endpoint = f"https://{R2_ACCOUNT_ID}.r2.cloudflarestorage.com" s3 = boto3.client( "s3", endpoint_url=r2_endpoint, aws_access_key_id=R2_ACCESS_KEY, aws_secret_access_key=R2_SECRET_KEY, config=Config(signature_version="s3v4") ) logger.info(f"Uploading compressed cache to R2 bucket '{R2_BUCKET}'...") s3.upload_file(COMPRESSED_DUMP_PATH, R2_BUCKET, "vault/redis/dump.rdb.gz") logger.info("Successfully updated Cloudflare R2 state swap partition!") return True except Exception as e: logger.error(f"Failed to sync with R2 swap partition: {e}") return False # ── GitHub Git-Archiver & Rollover Engine ────────────────────────────────────── def run_git_command(args: list, cwd: str = REDIS_DATA_DIR) -> str: """Executes a subprocess git command and returns the stdout result.""" result = subprocess.run( args, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=True ) return result.stdout.strip() def check_github_repo_size(repo_name: str) -> int: """Queries GitHub REST API to get repository size in Kilobytes.""" if not (GITHUB_PAT and GITHUB_USER): return 0 try: import urllib.request import json url = f"https://api.github.com/repos/{GITHUB_USER}/{repo_name}" req = urllib.request.Request(url) req.add_header("Authorization", f"token {GITHUB_PAT}") req.add_header("Accept", "application/vnd.github.v3+json") req.add_header("User-Agent", "SAOS-Backup-Daemon") with urllib.request.urlopen(req, timeout=8) as response: data = json.loads(response.read().decode()) size_kb = data.get("size", 0) logger.info(f"Active GitHub backup repo size: {size_kb} KB ({size_kb / 1024:.2f} MB)") return size_kb except Exception as e: logger.error(f"Failed to query repository size: {e}") return 0 def create_new_github_repo(repo_name: str) -> bool: """Programmatically creates a new private GitHub repository via REST API.""" logger.info(f"Creating fresh private GitHub repository: '{repo_name}'...") try: import urllib.request import json url = "https://api.github.com/user/repos" payload = json.dumps({ "name": repo_name, "private": True, "description": "SAOS Database Swap & Transcripts cold-archive partition.", "auto_init": False }).encode("utf-8") req = urllib.request.Request(url, data=payload) req.add_header("Authorization", f"token {GITHUB_PAT}") req.add_header("Content-Type", "application/json") req.add_header("Accept", "application/vnd.github.v3+json") req.add_header("User-Agent", "SAOS-Backup-Daemon") with urllib.request.urlopen(req, timeout=10) as response: if response.status == 201: logger.info(f"Successfully generated new private repository '{repo_name}'!") return True return False except Exception as e: logger.critical(f"GitHub repository generation failed: {e}") return False def sync_to_github(): """Commits and pushes the latest database snapshot to the active private repository.""" global ACTIVE_REPO_NAME if not (GITHUB_PAT and GITHUB_USER): logger.warning("GitHub credentials missing. Skipping Git-Archiver backup.") return # Check if active repo has filled up past 800MB (819,200 KB) current_size_kb = check_github_repo_size(ACTIVE_REPO_NAME) if current_size_kb >= 800000: logger.warning(f"Active repository '{ACTIVE_REPO_NAME}' is nearing 800MB limit. Running rollover...") # Increment index try: parts = ACTIVE_REPO_NAME.split("-part-") base = parts[0] num = int(parts[1]) if len(parts) > 1 else 1 new_repo_name = f"{base}-part-{num + 1}" except Exception: new_repo_name = f"{ACTIVE_REPO_NAME}-part-2" # Create new repo and swap remote if create_new_github_repo(new_repo_name): ACTIVE_REPO_NAME = new_repo_name # Re-initialize git origin try: run_git_command(["git", "remote", "remove", "origin"]) except Exception: pass origin_url = f"https://{GITHUB_USER}:{GITHUB_PAT}@github.com/{GITHUB_USER}/{new_repo_name}.git" run_git_command(["git", "remote", "add", "origin", origin_url]) logger.info(f"Swarm backup targets successfully rolled over to new remote repository: {new_repo_name}") if not os.path.exists(COMPRESSED_DUMP_PATH): return try: # Initialize Git repo inside data directory if not already created if not os.path.exists(os.path.join(REDIS_DATA_DIR, ".git")): logger.info("Initializing Git repository inside /data directory...") run_git_command(["git", "init"]) run_git_command(["git", "config", "user.name", "saos-backup-daemon"]) run_git_command(["git", "config", "user.email", "saos-daemon@olympus.internal"]) # Add secure remote origin_url = f"https://{GITHUB_USER}:{GITHUB_PAT}@github.com/{GITHUB_USER}/{ACTIVE_REPO_NAME}.git" run_git_command(["git", "remote", "add", "origin", origin_url]) # Commit and force-push the snapshot logger.info("Archiving snapshot to private GitHub repository...") shutil.copy2(COMPRESSED_DUMP_PATH, os.path.join(REDIS_DATA_DIR, "dump.rdb.gz")) # Write small history commit run_git_command(["git", "add", "dump.rdb.gz"]) timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") run_git_command(["git", "commit", "-m", f"SAOS state sync snapshot: {timestamp} [skip ci]"]) # Push to main run_git_command(["git", "branch", "-M", "main"]) run_git_command(["git", "push", "-f", "origin", "main"]) logger.info("GitHub Git-Archiver state commit successful!") except Exception as e: logger.error(f"Git-Archiver sync failed: {e}") # ── Daemon Entrypoint Loop ──────────────────────────────────────────────────── if __name__ == "__main__": logger.info("SAOS Cognitive Backup Sync Daemon is starting...") # Sleep 30 seconds on initial boot to let Redis populate dumps time.sleep(30) while True: logger.info("Triggering scheduled cognitive backup sequence...") # 1. Compress active database dump compressed_ok = compress_dump() if compressed_ok: # 2. Upload compressed dump to Cloudflare R2 independently upload_to_r2() # 3. Push compressed dump to Private GitHub Archiver independently sync_to_github() # Run every 5 minutes (300 seconds) time.sleep(300)