Upload 21 files

app/admin/router.py

@@ -27,7 +27,7 @@ import secrets
 from app.database import get_db, DB_PATH, integrity_check
 from app.backup import (
     create_snapshot, create_compressed_snapshot, create_vacuum_snapshot,
-    upload_to_hf_bucket, list_backups, restore_from_hf_bucket
+    upload_to_hf_bucket, list_backups, restore_from_hf_bucket, diagnose_hf_setup
 )
 
 router = APIRouter(prefix="/admin", tags=["admin"])
@@ -165,11 +165,13 @@ async def run_query(request: Request, sql: str = Form(...), _=Depends(verify_adm
 async def backups_page(request: Request, _=Depends(verify_admin)):
     backup_info = list_backups()
     health = integrity_check()
+    hf_diagnosis = diagnose_hf_setup()
     
     return templates.TemplateResponse("backups.html", {
         "request": request,
         "backups": backup_info,
         "health": health,
+        "hf_diagnosis": hf_diagnosis,
     })
 
 @router.post("/backups/create")
@@ -185,6 +187,8 @@ async def create_backup(request: Request, backup_type: str = Form("snapshot"), _
     else:
         result = {"status": "error", "message": f"Unknown type: {backup_type}"}
     
+    # Store result in session/cookie for display
+    # For simplicity, we just redirect - the result is logged
     return RedirectResponse(url="/admin/backups", status_code=303)
 
 @router.get("/backups/download/{filename}")
@@ -213,3 +217,20 @@ async def export_table(table_name: str, format: str = "json", _=Depends(verify_a
     conn = get_db()
     rows = conn.execute(f"SELECT * FROM [{table_name}]").fetchall()
     return [dict(r) for r in rows]
+
+# ── DIAGNOSTICS ──
+@router.get("/api/diagnostics")
+async def api_diagnostics(_=Depends(verify_admin)):
+    """Run HF diagnostics - check credentials and repo access."""
+    return diagnose_hf_setup()
+
+@router.post("/api/backup/upload")
+async def api_backup_upload(_=Depends(verify_admin)):
+    """Trigger HF bucket upload via API."""
+    return upload_to_hf_bucket()
+
+@router.post("/api/backup/test")
+async def api_backup_test(_=Depends(verify_admin)):
+    """Test backup creation and return result."""
+    result = create_compressed_snapshot()
+    return result

app/admin/templates/backups.html

@@ -7,6 +7,23 @@
         <a href="/admin/dashboard" class="btn-secondary">← Back to Dashboard</a>
     </div>
     
+    <!-- HF Diagnostics -->
+    <div class="card {% if '❌' in hf_diagnosis.overall_status %}error-card{% endif %}">
+        <h2>🔧 HF Setup Diagnostics</h2>
+        <div class="diagnostics">
+            {% for check in hf_diagnosis.checks %}
+            <div class="diag-item">
+                <span class="diag-name">{{ check.name }}</span>
+                <span class="diag-status">{{ check.status }}</span>
+                <span class="diag-value">{{ check.value }}</span>
+            </div>
+            {% endfor %}
+            <div class="diag-overall">
+                <strong>{{ hf_diagnosis.overall_status }}</strong>
+            </div>
+        </div>
+    </div>
+    
     <!-- Health Status -->
     <div class="card">
         <h2>Database Status</h2>
@@ -53,7 +70,11 @@
     <!-- Local Snapshots -->
     <div class="card">
         <h2>📁 Local Snapshots</h2>
-        <p class="info-text">Directory: <code>{{ backups.snapshot_dir }}</code></p>
+        <p class="info-text">
+            Directory: <code>{{ backups.snapshot_dir }}</code><br>
+            HF Bucket Repo: <code>{{ backups.hf_bucket_repo }}</code><br>
+            HF Token: <code>{% if backups.hf_token_set %}✅ Set{% else %}❌ Not Set{% endif %}</code>
+        </p>
         
         {% if backups.local_snapshots %}
         <table class="data-table">
@@ -83,6 +104,17 @@
         {% endif %}
     </div>
     
+    <!-- Quick API Test -->
+    <div class="card">
+        <h2>🧪 Quick Test</h2>
+        <p class="info-text">Use these API endpoints to test backup functionality:</p>
+        <div class="quick-queries">
+            <code>GET /admin/api/diagnostics</code> - Check HF setup<br>
+            <code>POST /admin/api/backup/test</code> - Test backup creation<br>
+            <code>POST /admin/api/backup/upload</code> - Upload to HF Bucket
+        </div>
+    </div>
+    
     <!-- Backup Strategy Info -->
     <div class="card">
         <h2>📚 3-Layer Backup Strategy</h2>

app/backup.py

@@ -1,6 +1,6 @@
 """
-Backup Module — 3-Layer Backup Strategy
-────────────────────────────────────────
+Backup Module — 3-Layer Backup Strategy (FIXED)
+────────────────────────────────────────────────
 Layer 1: Litestream → local file replica (every 1s, automatic)
 Layer 2: Python sqlite3.backup() → snapshot to /tmp (on-demand + scheduled)
 Layer 3: Upload snapshot to HF Bucket (persistent, survives restarts)
@@ -20,7 +20,7 @@ import shutil
 import logging
 from datetime import datetime, timedelta
 from pathlib import Path
-from huggingface_hub import HfApi, hf_hub_download
+from huggingface_hub import HfApi, hf_hub_download, create_repo, repo_exists
 from contextlib import contextmanager
 
 logger = logging.getLogger(__name__)
@@ -28,12 +28,16 @@ logger = logging.getLogger(__name__)
 DB_PATH = "/tmp/data/app.db"
 BACKUP_DIR = "/tmp/data/backups"
 SNAPSHOT_DIR = "/tmp/data/snapshots"
+RESTORE_DIR = "/tmp/data/restore"
 
 HF_TOKEN = os.environ.get("HF_TOKEN", "")
-HF_BUCKET_REPO = os.environ.get("HF_BUCKET_REPO", "")  # "username/bucket-name"
+HF_BUCKET_REPO = os.environ.get("HF_BUCKET_REPO", "")  # "username/repo-name"
 
+# Ensure all directories exist
 os.makedirs(BACKUP_DIR, exist_ok=True)
 os.makedirs(SNAPSHOT_DIR, exist_ok=True)
+os.makedirs(RESTORE_DIR, exist_ok=True)
+os.makedirs(os.path.dirname(DB_PATH), exist_ok=True)
 
 
 # ═══════════════════════════════════════════════════════
@@ -50,7 +54,10 @@ def create_snapshot(label: str = "manual") -> dict:
     Source: https://docs.python.org/3/library/sqlite3.html#sqlite3.Connection.backup
     """
     if not os.path.exists(DB_PATH):
-        return {"status": "error", "message": "Database not found"}
+        logger.warning(f"Database not found at {DB_PATH}, creating empty database first")
+        # Create empty database with schema
+        from app.database import init_db
+        init_db()
     
     timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
     snapshot_name = f"app_{label}_{timestamp}.db"
@@ -94,29 +101,53 @@ def create_compressed_snapshot() -> dict:
     
     Source: https://litestream.io/alternatives/cron/
     """
-    result = create_snapshot(label="compressed")
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    snapshot_name = f"app_compressed_{timestamp}.db.gz"
+    gz_path = os.path.join(SNAPSHOT_DIR, snapshot_name)
+    
+    # Create uncompressed snapshot first (temp)
+    result = create_snapshot(label="temp_for_compress")
     if result["status"] != "success":
         return result
     
-    snapshot_path = result["path"]
-    gz_path = snapshot_path + ".gz"
-    
-    with open(snapshot_path, 'rb') as f_in:
-        with gzip.open(gz_path, 'wb') as f_out:
-            shutil.copyfileobj(f_in, f_out)
-    
-    # Remove uncompressed version
-    os.remove(snapshot_path)
-    
-    gz_size = os.path.getsize(gz_path)
-    result["compressed_path"] = gz_path
-    result["compressed_size_bytes"] = gz_size
-    result["compressed_size_mb"] = round(gz_size / (1024 * 1024), 2)
-    result["compression_ratio"] = round(
-        (1 - gz_size / result["size_bytes"]) * 100, 1
-    )
+    temp_snapshot_path = result["path"]
+    original_size = result["size_bytes"]
     
-    return result
+    try:
+        # Compress directly to final location
+        with open(temp_snapshot_path, 'rb') as f_in:
+            with gzip.open(gz_path, 'wb') as f_out:
+                shutil.copyfileobj(f_in, f_out)
+        
+        # Remove temp uncompressed file
+        if os.path.exists(temp_snapshot_path):
+            os.remove(temp_snapshot_path)
+        
+        gz_size = os.path.getsize(gz_path)
+        
+        _log_backup("compressed_snapshot", gz_path, gz_size, "success")
+        
+        logger.info(f"✅ Compressed snapshot created: {snapshot_name} ({gz_size} bytes, {round((1 - gz_size/original_size)*100, 1)}% reduction)")
+        
+        return {
+            "status": "success",
+            "file": snapshot_name,
+            "path": gz_path,  # This is now the .gz file
+            "size_bytes": original_size,
+            "size_mb": round(original_size / (1024 * 1024), 2),
+            "compressed_size_bytes": gz_size,
+            "compressed_size_mb": round(gz_size / (1024 * 1024), 2),
+            "compression_ratio": round((1 - gz_size / original_size) * 100, 1),
+        }
+    except Exception as e:
+        # Clean up on failure
+        if os.path.exists(temp_snapshot_path):
+            os.remove(temp_snapshot_path)
+        if os.path.exists(gz_path):
+            os.remove(gz_path)
+        _log_backup("compressed_snapshot", gz_path, 0, "error", str(e))
+        logger.error(f"❌ Compressed snapshot failed: {e}")
+        return {"status": "error", "message": str(e)}
 
 
 def create_vacuum_snapshot() -> dict:
@@ -158,6 +189,47 @@ def create_vacuum_snapshot() -> dict:
 #  LAYER 3: HF Bucket Sync — Persistent Remote Storage
 # ═══════════════════════════════════════════════════════
 
+def ensure_hf_repo_exists() -> dict:
+    """
+    Ensure the HF dataset repository exists.
+    Creates it if it doesn't exist.
+    
+    Returns status dict.
+    """
+    if not HF_TOKEN:
+        return {"status": "error", "message": "HF_TOKEN not set"}
+    
+    if not HF_BUCKET_REPO:
+        return {"status": "error", "message": "HF_BUCKET_REPO not set"}
+    
+    try:
+        api = HfApi(token=HF_TOKEN)
+        
+        # Check if repo exists
+        try:
+            if repo_exists(repo_id=HF_BUCKET_REPO, repo_type="dataset", token=HF_TOKEN):
+                logger.info(f"✅ HF repo exists: {HF_BUCKET_REPO}")
+                return {"status": "success", "message": "Repo exists"}
+        except Exception:
+            pass
+        
+        # Create repo if it doesn't exist
+        logger.info(f"Creating HF dataset repo: {HF_BUCKET_REPO}")
+        create_repo(
+            repo_id=HF_BUCKET_REPO,
+            repo_type="dataset",
+            private=True,
+            exist_ok=True,
+            token=HF_TOKEN
+        )
+        logger.info(f"✅ Created HF dataset repo: {HF_BUCKET_REPO}")
+        return {"status": "success", "message": "Repo created"}
+        
+    except Exception as e:
+        logger.error(f"❌ Failed to ensure HF repo exists: {e}")
+        return {"status": "error", "message": str(e)}
+
+
 def upload_to_hf_bucket(local_path: str = None) -> dict:
     """
     Upload a database snapshot to HuggingFace Bucket.
@@ -166,31 +238,46 @@ def upload_to_hf_bucket(local_path: str = None) -> dict:
     
     Source: https://huggingface.co/docs/huggingface_hub/guides/upload
     """
+    # Check credentials
     if not HF_TOKEN or not HF_BUCKET_REPO:
-        return {"status": "error", "message": "HF_TOKEN or HF_BUCKET_REPO not set"}
+        msg = f"HF_TOKEN={'set' if HF_TOKEN else 'NOT SET'}, HF_BUCKET_REPO={'set' if HF_BUCKET_REPO else 'NOT SET'}"
+        logger.error(f"❌ HF credentials missing: {msg}")
+        return {"status": "error", "message": f"Credentials not configured: {msg}"}
     
-    api = HfApi(token=HF_TOKEN)
+    # Ensure repo exists
+    repo_status = ensure_hf_repo_exists()
+    if repo_status["status"] != "success":
+        return repo_status
     
     # Create snapshot if not provided
     if local_path is None:
         result = create_compressed_snapshot()
         if result["status"] != "success":
             return result
-        local_path = result.get("compressed_path", result["path"])
+        local_path = result["path"]  # Now correctly points to .gz file
+    
+    # Verify file exists
+    if not os.path.exists(local_path):
+        return {"status": "error", "message": f"Local file not found: {local_path}"}
     
     filename = os.path.basename(local_path)
-    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    file_size = os.path.getsize(local_path)
     
     try:
+        api = HfApi(token=HF_TOKEN)
+        
+        logger.info(f"📤 Uploading to HF Bucket: {filename} ({file_size} bytes)")
+        
         # Upload latest (overwrites)
         api.upload_file(
             path_or_fileobj=local_path,
-            path_in_repo=f"db-backups/latest.db.gz",
+            path_in_repo="db-backups/latest.db.gz",
             repo_id=HF_BUCKET_REPO,
-            repo_type="dataset",  # Use dataset repo as bucket
+            repo_type="dataset",
         )
         
-        # Also keep a timestamped copy (rolling 7-day)
+        # Also keep a timestamped copy (rolling archive)
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
         api.upload_file(
             path_or_fileobj=local_path,
             path_in_repo=f"db-backups/archive/{filename}",
@@ -198,16 +285,16 @@ def upload_to_hf_bucket(local_path: str = None) -> dict:
             repo_type="dataset",
         )
         
-        file_size = os.path.getsize(local_path)
         _log_backup("hf_bucket", f"db-backups/{filename}", file_size, "success")
         
         logger.info(f"✅ Uploaded to HF Bucket: {filename}")
-        return {"status": "success", "remote_path": f"db-backups/{filename}"}
+        return {"status": "success", "remote_path": f"db-backups/{filename}", "size_bytes": file_size}
         
     except Exception as e:
-        _log_backup("hf_bucket", local_path, 0, "error", str(e))
-        logger.error(f"❌ HF Bucket upload failed: {e}")
-        return {"status": "error", "message": str(e)}
+        error_msg = str(e)
+        _log_backup("hf_bucket", local_path, 0, "error", error_msg)
+        logger.error(f"❌ HF Bucket upload failed: {error_msg}")
+        return {"status": "error", "message": error_msg}
 
 
 def restore_from_hf_bucket() -> dict:
@@ -219,26 +306,31 @@ def restore_from_hf_bucket() -> dict:
     """
     if not HF_TOKEN or not HF_BUCKET_REPO:
         logger.warning("⚠️ HF credentials not set, skipping restore")
-        return {"status": "skipped", "message": "No HF credentials"}
+        return {"status": "skipped", "message": "No HF credentials configured"}
     
-    if os.path.exists(DB_PATH):
+    if os.path.exists(DB_PATH) and os.path.getsize(DB_PATH) > 0:
         logger.info("✅ Database already exists, skipping restore")
         return {"status": "skipped", "message": "DB already exists"}
     
+    # Ensure restore directory exists
+    os.makedirs(RESTORE_DIR, exist_ok=True)
+    os.makedirs(os.path.dirname(DB_PATH), exist_ok=True)
+    
     try:
-        api = HfApi(token=HF_TOKEN)
+        logger.info(f"📥 Attempting to restore from HF Bucket: {HF_BUCKET_REPO}")
         
         # Download latest backup
         local_gz = hf_hub_download(
             repo_id=HF_BUCKET_REPO,
             filename="db-backups/latest.db.gz",
             repo_type="dataset",
-            local_dir="/tmp/data/restore",
+            local_dir=RESTORE_DIR,
             token=HF_TOKEN,
         )
         
+        logger.info(f"✅ Downloaded backup: {local_gz}")
+        
         # Decompress
-        os.makedirs(os.path.dirname(DB_PATH), exist_ok=True)
         with gzip.open(local_gz, 'rb') as f_in:
             with open(DB_PATH, 'wb') as f_out:
                 shutil.copyfileobj(f_in, f_out)
@@ -257,8 +349,14 @@ def restore_from_hf_bucket() -> dict:
             return {"status": "error", "message": f"Integrity check failed: {result[0]}"}
             
     except Exception as e:
-        logger.info(f"📦 No backup found in HF Bucket: {e}")
-        return {"status": "not_found", "message": str(e)}
+        error_msg = str(e)
+        # Check if it's a "file not found" error (first run)
+        if "404" in error_msg or "not found" in error_msg.lower():
+            logger.info("📦 No backup found in HF Bucket (this is normal for first run)")
+            return {"status": "not_found", "message": "No backup exists yet - this is normal for first run"}
+        else:
+            logger.error(f"❌ Restore from HF Bucket failed: {error_msg}")
+            return {"status": "error", "message": error_msg}
 
 
 # ═══════════════════════════════════════════════════════
@@ -271,22 +369,41 @@ async def scheduled_backup_task():
     Creates snapshot + uploads to HF Bucket.
     """
     import asyncio
+    
+    # Wait a bit before first backup (let app stabilize)
+    await asyncio.sleep(60)
+    
     while True:
-        await asyncio.sleep(1800)  # 30 minutes
         try:
-            upload_to_hf_bucket()
+            logger.info("⏰ Running scheduled backup...")
+            result = upload_to_hf_bucket()
+            if result["status"] == "success":
+                logger.info("✅ Scheduled backup completed successfully")
+            else:
+                logger.warning(f"⚠️ Scheduled backup completed with issues: {result.get('message', 'unknown')}")
+            
             cleanup_old_snapshots(max_age_hours=48)
         except Exception as e:
-            logger.error(f"Scheduled backup error: {e}")
+            logger.error(f"❌ Scheduled backup error: {e}")
+        
+        await asyncio.sleep(1800)  # 30 minutes
 
 
 def cleanup_old_snapshots(max_age_hours: int = 48):
     """Remove local snapshots older than max_age_hours."""
     cutoff = datetime.now() - timedelta(hours=max_age_hours)
+    cleaned = 0
     for f in Path(SNAPSHOT_DIR).glob("*.db*"):
-        if datetime.fromtimestamp(f.stat().st_mtime) < cutoff:
-            f.unlink()
-            logger.info(f"🗑️ Removed old snapshot: {f.name}")
+        try:
+            if datetime.fromtimestamp(f.stat().st_mtime) < cutoff:
+                f.unlink()
+                cleaned += 1
+                logger.info(f"🗑️ Removed old snapshot: {f.name}")
+        except Exception as e:
+            logger.warning(f"Failed to remove {f.name}: {e}")
+    
+    if cleaned > 0:
+        logger.info(f"🗑️ Cleaned up {cleaned} old snapshots")
 
 
 def list_backups() -> dict:
@@ -294,14 +411,22 @@ def list_backups() -> dict:
     local_files = []
     
     for f in sorted(Path(SNAPSHOT_DIR).glob("*.db*"), key=lambda x: x.stat().st_mtime, reverse=True):
-        local_files.append({
-            "name": f.name,
-            "size_bytes": f.stat().st_size,
-            "size_mb": round(f.stat().st_size / (1024 * 1024), 2),
-            "modified": datetime.fromtimestamp(f.stat().st_mtime).isoformat(),
-        })
+        try:
+            local_files.append({
+                "name": f.name,
+                "size_bytes": f.stat().st_size,
+                "size_mb": round(f.stat().st_size / (1024 * 1024), 2),
+                "modified": datetime.fromtimestamp(f.stat().st_mtime).isoformat(),
+            })
+        except Exception:
+            pass
     
-    return {"local_snapshots": local_files, "snapshot_dir": SNAPSHOT_DIR}
+    return {
+        "local_snapshots": local_files, 
+        "snapshot_dir": SNAPSHOT_DIR,
+        "hf_bucket_repo": HF_BUCKET_REPO if HF_BUCKET_REPO else "Not configured",
+        "hf_token_set": bool(HF_TOKEN),
+    }
 
 
 # ── Internal helper ──
@@ -315,5 +440,90 @@ def _log_backup(backup_type, file_path, file_size, status, error=None):
         )
         conn.commit()
         conn.close()
-    except:
-        pass  # Don't fail if logging fails
+    except Exception as e:
+        logger.warning(f"Failed to log backup event: {e}")
+
+
+# ═══════════════════════════════════════════════════════
+#  DIAGNOSTICS
+# ═══════════════════════════════════════════════════════
+
+def diagnose_hf_setup() -> dict:
+    """
+    Diagnose HF setup issues.
+    Run this to check if everything is configured correctly.
+    """
+    results = {
+        "checks": [],
+        "overall_status": "unknown"
+    }
+    
+    # Check HF_TOKEN
+    if HF_TOKEN:
+        results["checks"].append({
+            "name": "HF_TOKEN",
+            "status": "✅ SET",
+            "value": f"{HF_TOKEN[:8]}...{HF_TOKEN[-4:]}" if len(HF_TOKEN) > 12 else "***"
+        })
+    else:
+        results["checks"].append({
+            "name": "HF_TOKEN",
+            "status": "❌ NOT SET",
+            "value": "Environment variable HF_TOKEN is missing"
+        })
+    
+    # Check HF_BUCKET_REPO
+    if HF_BUCKET_REPO:
+        results["checks"].append({
+            "name": "HF_BUCKET_REPO",
+            "status": "✅ SET",
+            "value": HF_BUCKET_REPO
+        })
+    else:
+        results["checks"].append({
+            "name": "HF_BUCKET_REPO",
+            "status": "❌ NOT SET",
+            "value": "Environment variable HF_BUCKET_REPO is missing"
+        })
+    
+    # Try to verify token by accessing API
+    if HF_TOKEN:
+        try:
+            api = HfApi(token=HF_TOKEN)
+            user_info = api.whoami()
+            results["checks"].append({
+                "name": "Token Validation",
+                "status": "✅ VALID",
+                "value": f"Logged in as: {user_info.get('name', 'unknown')}"
+            })
+        except Exception as e:
+            results["checks"].append({
+                "name": "Token Validation",
+                "status": "❌ INVALID",
+                "value": str(e)
+            })
+    
+    # Check if repo exists/is accessible
+    if HF_TOKEN and HF_BUCKET_REPO:
+        try:
+            exists = repo_exists(repo_id=HF_BUCKET_REPO, repo_type="dataset", token=HF_TOKEN)
+            results["checks"].append({
+                "name": "Repository Check",
+                "status": "✅ EXISTS" if exists else "⚠️ WILL BE CREATED",
+                "value": HF_BUCKET_REPO
+            })
+        except Exception as e:
+            results["checks"].append({
+                "name": "Repository Check",
+                "status": "❌ ERROR",
+                "value": str(e)
+            })
+    
+    # Determine overall status
+    failed = [c for c in results["checks"] if "❌" in c["status"]]
+    if not failed:
+        results["overall_status"] = "✅ All checks passed"
+    else:
+        results["overall_status"] = f"❌ {len(failed)} check(s) failed"
+    
+    return results

static/admin.css

@@ -437,6 +437,49 @@ body {
     line-height: 1.5;
 }
 
+/* ═══════════════════════════════════════════
+   Diagnostics
+   ═══════════════════════════════════════════ */
+
+.diagnostics {
+    display: flex;
+    flex-direction: column;
+    gap: 0.5rem;
+}
+
+.diag-item {
+    display: grid;
+    grid-template-columns: 150px 100px 1fr;
+    gap: 1rem;
+    padding: 0.75rem;
+    background: var(--background);
+    border-radius: 6px;
+    align-items: center;
+}
+
+.diag-name {
+    font-weight: 600;
+    color: var(--text-muted);
+}
+
+.diag-status {
+    font-weight: 600;
+}
+
+.diag-value {
+    font-family: 'JetBrains Mono', 'Fira Code', monospace;
+    font-size: 0.85rem;
+    color: var(--text-muted);
+    word-break: break-all;
+}
+
+.diag-overall {
+    margin-top: 1rem;
+    padding-top: 1rem;
+    border-top: 1px solid var(--border);
+    text-align: center;
+}
+
 /* ═══════════════════════════════════════════
    Footer
    ═══════════════════════════════════════════ */