deploy: auto-deploy 13:59:40

.gitignore

@@ -116,6 +116,7 @@ Thumbs.db
 
 logs/
 *.log
+persistence/
 
 *.db
 *.sqlite3

app/api/v1/router.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 from fastapi import APIRouter
 
-from app.api.v1 import auth, batch, chat, code_executor, convert, database, embeddings, reconcile, scraper, semantic_router, sql_validator, system, token_counter, token_generator, vector_stores, web_search
+from app.api.v1 import auth, batch, chat, code_executor, convert, database, embeddings, reconcile, scraper, semantic_router, sql_validator, system, token_counter, token_generator, vector_stores, web_search, webhook_socket
 from app.api.verify import router as verify_router
 
 api_v1_router = APIRouter()
@@ -23,3 +23,4 @@ api_v1_router.include_router(token_counter.router, tags=["Token Counter"])
 api_v1_router.include_router(token_generator.router, tags=["Token Generator"])
 api_v1_router.include_router(chat.router, tags=["Chat"])
 api_v1_router.include_router(vector_stores.router, tags=["Vector Stores"])
+api_v1_router.include_router(webhook_socket.router, tags=["Webhook / Socket"])

app/api/v1/system.py

@@ -1,10 +1,14 @@
 from __future__ import annotations
 
+import io
 import platform
+import tarfile
 import time
 from datetime import datetime, timezone
+from pathlib import Path
 
-from fastapi import APIRouter, Depends
+from fastapi import APIRouter, Depends, File, HTTPException, UploadFile, status
+from fastapi.responses import StreamingResponse
 
 from app.api.deps import get_extraction_service, require_auth
 from app.config import get_settings
@@ -18,6 +22,7 @@ from app.core.constants import (
     TEXT_EXTENSIONS,
     WEB_EXTENSIONS,
 )
+from app.core.logger import get_logger
 from app.models.schemas import (
     HealthResponse,
     InfoResponse,
@@ -29,6 +34,7 @@ from app.services.extraction_service import ExtractionService
 router = APIRouter()
 _settings = get_settings()
 _START_TIME = time.time()
+_logger = get_logger(__name__)
 
 
 @router.get("/health", response_model=HealthResponse, summary="Health check")
@@ -108,3 +114,74 @@ async def list_spacy_labels(
             "phone": {"source_type": "regex", "pattern": r"\b\d{3}-\d{3}-\d{4}\b"},
         },
     )
+
+
+def _build_archive(data_dir: Path) -> io.BytesIO:
+    buf = io.BytesIO()
+    with tarfile.open(fileobj=buf, mode="w:gz") as tar:
+        for path in sorted(data_dir.rglob("*")):
+            if path.is_file():
+                arcname = path.relative_to(data_dir.parent)
+                tar.add(str(path), arcname=str(arcname))
+    buf.seek(0)
+    return buf
+
+
+def _get_data_dir() -> Path:
+    raw = _settings.data_dir
+    p = Path(raw)
+    if not p.is_absolute():
+        p = Path.cwd() / p
+    return p.resolve()
+
+
+@router.get("/backup", summary="Download a full data backup archive")
+async def download_backup(token: str = Depends(require_auth)):
+    data_dir = _get_data_dir()
+    if not data_dir.is_dir():
+        raise HTTPException(status_code=404, detail="Data directory not found")
+    try:
+        archive = _build_archive(data_dir)
+    except Exception as exc:
+        _logger.error("Backup creation failed: %s", exc)
+        raise HTTPException(status_code=500, detail=f"Backup failed: {exc}")
+
+    return StreamingResponse(
+        archive,
+        media_type="application/gzip",
+        headers={"Content-Disposition": "attachment; filename=backup.tar.gz"},
+    )
+
+
+@router.post("/backup/restore", summary="Upload and restore a data backup archive")
+async def upload_and_restore(
+    file: UploadFile = File(...),
+    token: str = Depends(require_auth),
+):
+    data_dir = _get_data_dir()
+    data_dir.mkdir(parents=True, exist_ok=True)
+
+    try:
+        content = await file.read()
+    except Exception as exc:
+        raise HTTPException(status_code=400, detail=f"Failed to read upload: {exc}")
+
+    if not content:
+        raise HTTPException(status_code=400, detail="Empty file")
+
+    restored = 0
+    try:
+        with tarfile.open(fileobj=io.BytesIO(content), mode="r:gz") as tar:
+            for member in tar.getmembers():
+                if member.isfile():
+                    tar.extract(member, path=data_dir.parent)
+                    restored += 1
+    except tarfile.TarError as exc:
+        raise HTTPException(status_code=400, detail=f"Invalid archive: {exc}")
+
+    return {
+        "success": True,
+        "message": f"Restored {restored} files to {data_dir}",
+        "files_restored": restored,
+        "data_dir": str(data_dir),
+    }

app/api/v1/webhook_socket.py

@@ -0,0 +1,250 @@
+from __future__ import annotations
+
+import asyncio
+import json
+
+from fastapi import APIRouter, Depends, Request, WebSocket, WebSocketDisconnect
+
+from app.api.deps import require_auth
+from app.models.schemas import (
+    ChannelCreateRequest,
+    ChannelCreateResponse,
+    ChannelDeleteResponse,
+    ChannelInfoResponse,
+    ChannelListItem,
+    ChannelListResponse,
+    WebhookResponse,
+    WebhookSocketStatsResponse,
+)
+from app.services.webhook_socket_service import get_manager, verify_signature
+
+router = APIRouter()
+manager = get_manager()
+
+
+@router.post("/channels", response_model=ChannelCreateResponse, summary="Create a webhook channel")
+async def create_channel(
+    body: ChannelCreateRequest,
+    request: Request,
+    token: str = Depends(require_auth),
+):
+    if body.channel_id and manager.get_channel(body.channel_id):
+        from fastapi import HTTPException
+        raise HTTPException(status_code=409, detail=f"Channel '{body.channel_id}' already exists")
+
+    ch = manager.create_channel(
+        channel_id=body.channel_id,
+        secret=body.secret,
+        buffer_size=body.buffer_size,
+    )
+
+    host = request.headers.get("host", "localhost:7860")
+    scheme = request.headers.get("x-forwarded-proto", "http")
+    ws_scheme = "wss" if scheme == "https" else "ws"
+
+    return ChannelCreateResponse(
+        channel_id=ch.channel_id,
+        webhook_url=f"{scheme}://{host}/api/v1/webhook/{ch.channel_id}",
+        ws_url=f"{ws_scheme}://{host}/api/v1/ws/{ch.channel_id}",
+        secret=ch.secret,
+        buffer_size=ch.buffer_size,
+    )
+
+
+@router.get("/channels", response_model=ChannelListResponse, summary="List all webhook channels")
+async def list_channels(
+    token: str = Depends(require_auth),
+):
+    channels = []
+    for cid, ch in manager.channels.items():
+        channels.append(ChannelListItem(
+            channel_id=cid,
+            subscribers=len(ch.subscribers),
+            messages=ch.message_count,
+            buffered=len(ch.history),
+            created_at=ch.created_at,
+            last_activity=ch.last_activity,
+        ))
+    return ChannelListResponse(channels=channels)
+
+
+@router.get("/channels/{channel_id}", response_model=ChannelInfoResponse, summary="Get channel info")
+async def channel_info(
+    channel_id: str,
+    token: str = Depends(require_auth),
+):
+    from fastapi import HTTPException
+    ch = manager.get_channel(channel_id)
+    if not ch:
+        raise HTTPException(status_code=404, detail="Channel not found")
+    return ChannelInfoResponse(
+        channel_id=ch.channel_id,
+        subscribers=len(ch.subscribers),
+        messages=ch.message_count,
+        buffered=len(ch.history),
+        buffer_size=ch.buffer_size,
+        has_secret=ch.secret is not None,
+        created_at=ch.created_at,
+        last_activity=ch.last_activity,
+    )
+
+
+@router.delete("/channels/{channel_id}", response_model=ChannelDeleteResponse, summary="Delete a channel")
+async def delete_channel(
+    channel_id: str,
+    token: str = Depends(require_auth),
+):
+    from fastapi import HTTPException
+    if manager.delete_channel(channel_id):
+        return ChannelDeleteResponse(deleted=channel_id)
+    raise HTTPException(status_code=404, detail="Channel not found")
+
+
+@router.post("/webhook/{channel_id}", response_model=WebhookResponse, summary="Send webhook payload to channel")
+async def handle_webhook(
+    channel_id: str,
+    request: Request,
+):
+    from fastapi import HTTPException
+    ch = manager.get_channel(channel_id)
+    if not ch:
+        raise HTTPException(status_code=404, detail="Channel not found")
+
+    raw_body = await request.body()
+
+    if ch.secret:
+        sig_header = request.headers.get("X-Signature-256") or request.headers.get("X-Hub-Signature-256", "")
+        if not sig_header:
+            raise HTTPException(status_code=401, detail="Missing signature header")
+        if not verify_signature(ch.secret, raw_body, sig_header):
+            raise HTTPException(status_code=401, detail="Invalid signature")
+
+    content_type = request.headers.get("content-type", "")
+
+    if "json" in content_type:
+        try:
+            payload = json.loads(raw_body)
+        except json.JSONDecodeError:
+            payload = raw_body.decode(errors="replace")
+    elif "x-www-form-urlencoded" in content_type:
+        form = await request.form()
+        payload = dict(form)
+    else:
+        try:
+            payload = json.loads(raw_body)
+        except json.JSONDecodeError:
+            payload = raw_body.decode(errors="replace")
+
+    fwd_headers: dict[str, str] = {}
+    for h in ("X-GitHub-Event", "X-GitHub-Delivery", "X-Event-Type",
+              "X-Webhook-Name", "Content-Type", "User-Agent"):
+        if h in request.headers:
+            fwd_headers[h] = request.headers[h]
+
+    sent = await manager.publish(channel_id, payload, fwd_headers)
+
+    ch_ref = manager.get_channel(channel_id)
+    return WebhookResponse(
+        status="delivered",
+        channel=channel_id,
+        subscribers_notified=max(sent, 0),
+        message_id=ch_ref.message_count if ch_ref else 0,
+    )
+
+
+@router.post("/hook/{channel_id}", response_model=WebhookResponse, summary="Send webhook payload (short alias)")
+async def handle_webhook_short(
+    channel_id: str,
+    request: Request,
+):
+    return await handle_webhook(channel_id, request)
+
+
+@router.get("/ws/{channel_id}")
+async def websocket_endpoint(
+    channel_id: str,
+    websocket: WebSocket,
+    secret: str = "",
+):
+    ch = manager.get_channel(channel_id)
+    if not ch:
+        await websocket.accept()
+        await websocket.send_json({"event": "error", "message": "channel not found"})
+        await websocket.close(code=4404)
+        return
+
+    if ch.secret:
+        import hmac as hmac_mod
+        if not hmac_mod.compare_digest(secret, ch.secret):
+            await websocket.accept()
+            await websocket.send_json({"event": "error", "message": "unauthorized"})
+            await websocket.close(code=4401)
+            return
+
+    await websocket.accept()
+
+    q = manager.subscribe(channel_id, websocket)
+    if q is None:
+        await websocket.send_json({"event": "error", "message": "subscribe failed"})
+        await websocket.close()
+        return
+
+    await websocket.send_json({
+        "event": "connected",
+        "channel": channel_id,
+        "message": f"Listening on channel '{channel_id}'",
+        "buffered": len(ch.history),
+    })
+
+    async def forward_to_ws():
+        try:
+            while True:
+                try:
+                    msg = await asyncio.wait_for(q.get(), timeout=30)
+                except asyncio.TimeoutError:
+                    try:
+                        await websocket.send_json({"event": "ping"})
+                    except Exception:
+                        break
+                    continue
+                try:
+                    await websocket.send_json(msg)
+                except Exception:
+                    break
+        except asyncio.CancelledError:
+            pass
+
+    async def read_from_ws():
+        try:
+            while True:
+                try:
+                    data = await websocket.receive_text()
+                    try:
+                        msg_data = json.loads(data)
+                        if msg_data.get("type") == "ping":
+                            await websocket.send_json({"event": "pong"})
+                    except json.JSONDecodeError:
+                        pass
+                except WebSocketDisconnect:
+                    break
+        except asyncio.CancelledError:
+            pass
+
+    fwd_task = asyncio.create_task(forward_to_ws())
+    read_task = asyncio.create_task(read_from_ws())
+
+    try:
+        done, pending = await asyncio.wait(
+            [fwd_task, read_task], return_when=asyncio.FIRST_COMPLETED
+        )
+    finally:
+        fwd_task.cancel()
+        read_task.cancel()
+        manager.unsubscribe(channel_id, websocket)
+
+
+@router.get("/webhook-socket/stats", response_model=WebhookSocketStatsResponse, summary="Webhook/socket server stats")
+async def ws_stats(
+    token: str = Depends(require_auth),
+):
+    return manager.stats()

app/models/schemas.py

@@ -659,3 +659,63 @@ class DeleteResponse(BaseModel):
     deleted_count: int
     time_ms: float
     error: Optional[str] = None
+
+
+# ---------------------------------------------------------------------------
+# Webhook / Socket
+# ---------------------------------------------------------------------------
+
+class ChannelCreateRequest(BaseModel):
+    channel_id: Optional[str] = Field(None, min_length=1, max_length=64, description="Optional custom channel ID")
+    secret: Optional[str] = Field(None, min_length=1, description="HMAC secret for webhook verification")
+    buffer_size: Optional[int] = Field(None, ge=0, le=10000, description="Replay buffer size (0 = no replay)")
+
+
+class ChannelCreateResponse(BaseModel):
+    channel_id: str
+    webhook_url: str
+    ws_url: str
+    secret: Optional[str] = None
+    buffer_size: int
+
+
+class ChannelInfoResponse(BaseModel):
+    channel_id: str
+    subscribers: int
+    messages: int
+    buffered: int
+    buffer_size: int
+    has_secret: bool
+    created_at: float
+    last_activity: float
+
+
+class ChannelListItem(BaseModel):
+    channel_id: str
+    subscribers: int
+    messages: int
+    buffered: int
+    created_at: float
+    last_activity: float
+
+
+class ChannelListResponse(BaseModel):
+    channels: List[ChannelListItem]
+
+
+class ChannelDeleteResponse(BaseModel):
+    deleted: str
+
+
+class WebhookResponse(BaseModel):
+    status: str
+    channel: str
+    subscribers_notified: int
+    message_id: int
+
+
+class WebhookSocketStatsResponse(BaseModel):
+    channels: int
+    total_messages: int
+    total_subscribers: int
+    channels_detail: Dict[str, Dict[str, Any]]

app/services/webhook_socket_service.py

@@ -0,0 +1,168 @@
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import hmac
+import time
+import uuid
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+from app.core.logger import get_logger
+
+log = get_logger("webhook-socket")
+
+
+@dataclass
+class Channel:
+    channel_id: str
+    secret: Optional[str] = None
+    buffer_size: int = 0
+    created_at: float = field(default_factory=time.time)
+    message_count: int = 0
+    last_activity: float = field(default_factory=time.time)
+    subscribers: Dict[Any, asyncio.Queue] = field(default_factory=dict)
+    history: List[dict] = field(default_factory=list)
+
+
+class ChannelManager:
+    def __init__(self, default_buffer: int = 0):
+        self.channels: Dict[str, Channel] = {}
+        self.default_buffer = default_buffer
+        self.total_messages = 0
+
+    def create_channel(
+        self,
+        channel_id: Optional[str] = None,
+        secret: Optional[str] = None,
+        buffer_size: Optional[int] = None,
+    ) -> Channel:
+        ch_id = channel_id or uuid.uuid4().hex[:16]
+        buf = buffer_size if buffer_size is not None else self.default_buffer
+        ch = Channel(channel_id=ch_id, secret=secret, buffer_size=buf)
+        self.channels[ch_id] = ch
+        log.info("Channel created  id=%s  buffer=%d  secret=%s", ch_id, buf, "yes" if secret else "no")
+        return ch
+
+    def get_channel(self, channel_id: str) -> Optional[Channel]:
+        return self.channels.get(channel_id)
+
+    def delete_channel(self, channel_id: str) -> bool:
+        if channel_id in self.channels:
+            for q in self.channels[channel_id].subscribers.values():
+                q.put_nowait({"event": "channel_deleted", "channel": channel_id})
+            del self.channels[channel_id]
+            log.info("Channel deleted  id=%s", channel_id)
+            return True
+        return False
+
+    def subscribe(self, channel_id: str, ws: Any) -> Optional[asyncio.Queue]:
+        ch = self.channels.get(channel_id)
+        if not ch:
+            return None
+        q: asyncio.Queue = asyncio.Queue()
+        ch.subscribers[ws] = q
+        log.info(
+            "Subscriber joined  channel=%s  total_subs=%d",
+            channel_id, len(ch.subscribers),
+        )
+        for msg in ch.history:
+            q.put_nowait(msg)
+        return q
+
+    def unsubscribe(self, channel_id: str, ws: Any) -> None:
+        ch = self.channels.get(channel_id)
+        if ch and ws in ch.subscribers:
+            del ch.subscribers[ws]
+            log.info(
+                "Subscriber left   channel=%s  total_subs=%d",
+                channel_id, len(ch.subscribers),
+            )
+
+    async def publish(
+        self,
+        channel_id: str,
+        payload: Any,
+        headers: Optional[Dict[str, str]] = None,
+    ) -> int:
+        ch = self.channels.get(channel_id)
+        if not ch:
+            return -1
+
+        message = {
+            "event": "message",
+            "channel": channel_id,
+            "timestamp": time.time(),
+            "id": uuid.uuid4().hex[:12],
+            "payload": payload,
+            "headers": headers or {},
+        }
+
+        ch.message_count += 1
+        ch.last_activity = time.time()
+        self.total_messages += 1
+
+        if ch.buffer_size > 0:
+            ch.history.append(message)
+            while len(ch.history) > ch.buffer_size:
+                ch.history.pop(0)
+
+        dead: List[Any] = []
+        sent = 0
+        for ws, q in list(ch.subscribers.items()):
+            if getattr(ws, "closed", False):
+                dead.append(ws)
+                continue
+            await q.put(message)
+            sent += 1
+
+        for ws in dead:
+            del ch.subscribers[ws]
+
+        log.info(
+            "Published  channel=%s  subs=%d  msg_total=%d",
+            channel_id, sent, ch.message_count,
+        )
+        return sent
+
+    def stats(self) -> dict:
+        return {
+            "channels": len(self.channels),
+            "total_messages": self.total_messages,
+            "total_subscribers": sum(len(c.subscribers) for c in self.channels.values()),
+            "channels_detail": {
+                cid: {
+                    "subscribers": len(ch.subscribers),
+                    "messages": ch.message_count,
+                    "buffered": len(ch.history),
+                    "last_activity": ch.last_activity,
+                    "has_secret": ch.secret is not None,
+                }
+                for cid, ch in self.channels.items()
+            },
+        }
+
+
+def sign_payload(secret: str, raw_body: bytes) -> str:
+    return "sha256=" + hmac.new(secret.encode(), raw_body, hashlib.sha256).hexdigest()
+
+
+def verify_signature(secret: str, raw_body: bytes, signature: str) -> bool:
+    expected = sign_payload(secret, raw_body)
+    return hmac.compare_digest(expected, signature)
+
+
+_manager: Optional[ChannelManager] = None
+
+
+def get_manager() -> ChannelManager:
+    global _manager
+    if _manager is None:
+        _manager = ChannelManager()
+    return _manager
+
+
+def init_manager(default_buffer: int = 0) -> ChannelManager:
+    global _manager
+    _manager = ChannelManager(default_buffer=default_buffer)
+    return _manager

deploy.py

@@ -0,0 +1,959 @@
+#!/usr/bin/env python3
+"""
+deploy.py — Persistent data deployment for Hugging Face Spaces.
+
+Usage:
+    python deploy.py
+
+Flow:
+    1. Back up existing data from the deployed Space -> persistence/
+    2. Commit and push local changes to the Space repository
+    3. Wait for the Space to redeploy (poll /health)
+    4. Restore the backed-up data to the new deployment
+    5. Generate a comprehensive audit report
+"""
+
+import argparse
+import io
+import json
+import os
+import re
+import subprocess
+import sys
+import tarfile
+import time
+import urllib.error
+import urllib.request
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Optional
+
+# ─────────────────────────────────────────────────────────────
+# Console encoding detection
+# ─────────────────────────────────────────────────────────────
+
+_USE_UNICODE = True
+try:
+    "\u2713".encode(sys.stdout.encoding or "utf-8")
+except (UnicodeEncodeError, UnicodeDecodeError, LookupError):
+    _USE_UNICODE = False
+
+# ─────────────────────────────────────────────────────────────
+# ANSI colours & helpers
+# ─────────────────────────────────────────────────────────────
+
+GREEN = "\033[92m"
+RED = "\033[91m"
+YELLOW = "\033[93m"
+CYAN = "\033[96m"
+BOLD = "\033[1m"
+RESET = "\033[0m"
+
+
+def _c(colour: str, text: str) -> str:
+    return f"{colour}{text}{RESET}"
+
+
+def _ok(text: str) -> str:
+    return _c(GREEN, text)
+
+
+def _fail(text: str) -> str:
+    return _c(RED, text)
+
+
+def _warn(text: str) -> str:
+    return _c(YELLOW, text)
+
+
+def _info(text: str) -> str:
+    return _c(CYAN, text)
+
+
+def _bold(text: str) -> str:
+    return _c(BOLD, text)
+
+
+# Icon constants (must be defined outside f-string expressions for Python < 3.12)
+# Unicode vs ASCII fallbacks for Windows console compatibility
+if _USE_UNICODE:
+    _OK_ICON = _ok("\u2713")
+    _FAIL_ICON = _fail("\u2717")
+    _WARN_ICON = _warn("\u26A0")
+    _INFO_ICON = _info("\u25B8")
+    _BULLET = "\u2022"
+    _DASH = "\u2014"
+    _TABLE_TL = "\u2554"
+    _TABLE_TR = "\u2557"
+    _TABLE_BL = "\u255A"
+    _TABLE_BR = "\u255D"
+    _TABLE_H = "\u2550"
+    _TABLE_V = "\u2551"
+    _TABLE_TM = "\u2560"
+    _TABLE_BM = "\u255A"
+    _TABLE_ML = "\u251C"
+    _TABLE_MR = "\u2524"
+    _TABLE_MM = "\u253C"
+    _TABLE_MH = "\u2500"
+    _TABLE_MV = "\u2502"
+    _TABLE_TML = "\u255F"
+    _TABLE_TMR = "\u257E"
+else:
+    _OK_ICON = _ok("v")
+    _FAIL_ICON = _fail("x")
+    _WARN_ICON = _warn("!")
+    _INFO_ICON = _info(">")
+    _BULLET = "*"
+    _DASH = "-"
+    _TABLE_TL = "+"
+    _TABLE_TR = "+"
+    _TABLE_BL = "+"
+    _TABLE_BR = "+"
+    _TABLE_H = "="
+    _TABLE_V = "|"
+    _TABLE_TM = "+"
+    _TABLE_BM = "+"
+    _TABLE_ML = "+"
+    _TABLE_MR = "+"
+    _TABLE_MM = "+"
+    _TABLE_MH = "-"
+    _TABLE_MV = "|"
+    _TABLE_TML = "+"
+    _TABLE_TMR = "-"
+
+
+def _indicator(status: str) -> str:
+    mapping = {
+        "passed": _OK_ICON,
+        "failed": _FAIL_ICON,
+        "skipped": _WARN_ICON,
+        "running": _INFO_ICON,
+    }
+    return mapping.get(status, _DASH)
+
+
+def _timestamp() -> str:
+    return datetime.now(timezone.utc).strftime("%H:%M:%S")
+
+
+def _human_size(n_bytes: int) -> str:
+    for unit in ("B", "KB", "MB", "GB"):
+        if n_bytes < 1024:
+            return f"{n_bytes:.2f} {unit}"
+        n_bytes /= 1024
+    return f"{n_bytes:.2f} TB"
+
+
+def _human_duration(seconds: float) -> str:
+    if seconds < 60:
+        return f"{seconds:.1f}s"
+    mins = int(seconds // 60)
+    secs = int(seconds % 60)
+    return f"{mins}m {secs}s"
+
+
+# ─────────────────────────────────────────────────────────────
+# Environment loading
+# ─────────────────────────────────────────────────────────────
+
+
+def load_env(env_path: str = ".env") -> dict[str, str]:
+    env: dict[str, str] = {}
+    p = Path(env_path)
+    if not p.is_file():
+        return env
+    for line in p.read_text(encoding="utf-8").splitlines():
+        line = line.strip()
+        if not line or line.startswith("#"):
+            continue
+        m = re.match(r"^([A-Za-z_][A-Za-z_0-9]*)\s*=\s*(.*?)\s*$", line)
+        if not m:
+            continue
+        key = m.group(1)
+        val = m.group(2)
+        if val.startswith('"') and val.endswith('"'):
+            val = val[1:-1]
+        elif val.startswith("'") and val.endswith("'"):
+            val = val[1:-1]
+        env[key] = val
+    return env
+
+
+# ─────────────────────────────────────────────────────────────
+# HTTP helpers
+# ─────────────────────────────────────────────────────────────
+
+BACKUP_ARCHIVE = "backup.tar.gz"
+RESTORE_ENDPOINT = "/api/v1/system/backup/restore"
+BACKUP_ENDPOINT = "/api/v1/system/backup"
+HEALTH_ENDPOINT = "/health"
+HF_API_SPACES = "https://huggingface.co/api/spaces"
+
+
+def _build_url(base: str, path: str) -> str:
+    base = base.rstrip("/")
+    path = path.lstrip("/")
+    return f"{base}/{path}"
+
+
+def http_get(url: str, headers: dict[str, str], timeout: int = 120) -> tuple[int, bytes]:
+    req = urllib.request.Request(url, headers=headers, method="GET")
+    with urllib.request.urlopen(req, timeout=timeout) as resp:
+        return resp.status, resp.read()
+
+
+def http_get_stream(
+    url: str, headers: dict[str, str], dest: Path, timeout: int = 300
+) -> tuple[int, int]:
+    req = urllib.request.Request(url, headers=headers, method="GET")
+    with urllib.request.urlopen(req, timeout=timeout) as resp:
+        total = 0
+        with open(dest, "wb") as f:
+            while True:
+                chunk = resp.read(65536)
+                if not chunk:
+                    break
+                f.write(chunk)
+                total += len(chunk)
+        return resp.status, total
+
+
+def http_post_multipart(
+    url: str,
+    file_path: Path,
+    field_name: str,
+    headers: dict[str, str],
+    timeout: int = 300,
+) -> tuple[int, bytes]:
+    boundary = "----DeployBoundary" + hex(int(time.time() * 1e6))[2:]
+    data = io.BytesIO()
+
+    data.write(f"--{boundary}\r\n".encode())
+    data.write(
+        f'Content-Disposition: form-data; name="{field_name}"; filename="{file_path.name}"\r\n'.encode()
+    )
+    data.write(b"Content-Type: application/gzip\r\n\r\n")
+    data.write(file_path.read_bytes())
+    data.write(f"\r\n--{boundary}--\r\n".encode())
+
+    body = data.getvalue()
+    content_type = f"multipart/form-data; boundary={boundary}"
+
+    req_headers = {**headers, "Content-Type": content_type}
+    req = urllib.request.Request(url, data=body, headers=req_headers, method="POST")
+    with urllib.request.urlopen(req, timeout=timeout) as resp:
+        return resp.status, resp.read()
+
+
+def http_get_json(
+    url: str, headers: dict[str, str], timeout: int = 30
+) -> Optional[dict[str, Any]]:
+    try:
+        req = urllib.request.Request(url, headers=headers, method="GET")
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            return json.loads(resp.read().decode("utf-8"))
+    except Exception:
+        return None
+
+
+def http_head(url: str, timeout: int = 30) -> Optional[int]:
+    try:
+        req = urllib.request.Request(url, method="HEAD")
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            return resp.status
+    except urllib.error.URLError:
+        return None
+
+
+# ─────────────────────────────────────────────────────────────
+# Git helpers
+# ─────────────────────────────────────────────────────────────
+
+
+def _run_git(args: list[str], cwd: str | None = None) -> tuple[int, str]:
+    cmd = ["git"] + args
+    result = subprocess.run(cmd, capture_output=True, text=True, cwd=cwd)
+    return result.returncode, result.stdout.strip()
+
+
+def git_status(cwd: str) -> list[str]:
+    rc, out = _run_git(["status", "--porcelain"], cwd=cwd)
+    if rc != 0:
+        return []
+    lines = [line for line in out.split("\n") if line.strip()]
+    return lines
+
+
+def git_add_all(cwd: str) -> bool:
+    changes = git_status(cwd)
+    if not changes:
+        return True
+    paths = []
+    for line in changes:
+        fn = line[3:].strip() if len(line) > 3 else ""
+        if fn and fn.lower() != "nul":
+            paths.append(fn)
+    if not paths:
+        return True
+    rc, _ = _run_git(["add", "--"] + paths, cwd=cwd)
+    return rc == 0
+
+
+def git_commit(cwd: str, message: str) -> tuple[bool, str]:
+    rc, out = _run_git(["commit", "-m", message], cwd=cwd)
+    if rc == 0:
+        m = re.search(r"\[[^\]]+ ([a-f0-9]+)\]", out)
+        sha = m.group(1) if m else "unknown"
+        return True, sha
+    if "nothing to commit" in out.lower() or "no changes" in out.lower():
+        return True, "no-change"
+    return False, out
+
+
+def git_push(cwd: str, remote: str = "origin", branch: str = "main") -> tuple[bool, str]:
+    rc, out = _run_git(["push", remote, branch], cwd=cwd)
+    return rc == 0, out
+
+
+def _parse_hf_space_from_url(url: str) -> tuple[str, str]:
+    m = re.search(r"huggingface\.co/spaces/([^/]+)/([^/\s]+)", url)
+    if m:
+        return m.group(1), m.group(2).rstrip("/")
+    return "aetherbase", "llm-ready-data"
+
+
+def git_remote_url(cwd: str) -> str:
+    rc, out = _run_git(["remote", "get-url", "origin"], cwd=cwd)
+    if rc != 0:
+        return ""
+    return out
+
+
+def git_log(cwd: str, n: int = 3) -> str:
+    _, out = _run_git(["log", "--oneline", f"-{n}"], cwd=cwd)
+    return out
+
+
+# ─────────────────────────────────────────────────────────────
+# Core deployment logic
+# ─────────────────────────────────────────────────────────────
+
+
+class AuditReport:
+    def __init__(self):
+        self.start_time = time.time()
+        self.fields: dict[str, Any] = {
+            "backup_started": "",
+            "backup_completed": "",
+            "backup_files": 0,
+            "backup_size": 0,
+            "backup_status": "pending",
+            "commit_status": "pending",
+            "commit_hash": "",
+            "push_status": "pending",
+            "deploy_status": "pending",
+            "deploy_duration": "",
+            "restore_status": "pending",
+            "restore_files": 0,
+            "verification_status": "pending",
+            "errors": [],
+            "warnings": [],
+        }
+
+    def end(self):
+        self.fields["total_duration"] = _human_duration(time.time() - self.start_time)
+
+    def set(self, key: str, value: Any):
+        self.fields[key] = value
+
+    def error(self, msg: str):
+        self.fields["errors"].append(msg)
+
+    def warn(self, msg: str):
+        self.fields["warnings"].append(msg)
+
+    def _row(self, label: str, status: str, detail: str) -> str:
+        icon = _indicator(status)
+        return f"{_TABLE_MV} {label:<28} {icon:<2}    {detail:<30} {_TABLE_MV}"
+
+    def _sep(self) -> str:
+        return f"{_TABLE_ML}{_TABLE_MH * 28}{_TABLE_MM}{_TABLE_MH * 4}{_TABLE_MM}{_TABLE_MH * 32}{_TABLE_MR}"
+
+    def print(self):
+        self.end()
+        f = self.fields
+        err_count = len(f["errors"])
+        warn_count = len(f["warnings"])
+
+        start_dt = datetime.fromtimestamp(self.start_time).strftime("%c")
+
+        lines: list[str] = []
+        lines.append("")
+        lines.append(
+            f"{_TABLE_TL}{_TABLE_H * 16}{_TABLE_H * 16}{_TABLE_H * 18}{_TABLE_H * 20}"
+            f"{_TABLE_TR}"
+        )
+
+        title = "DEPLOYMENT AUDIT REPORT"
+        padding = (70 - len(title)) // 2
+        lines.append(
+            f"{_TABLE_V} {' ' * padding}{_bold(title)}{' ' * (70 - padding - len(title))} {_TABLE_V}"
+        )
+        lines.append(
+            f"{_TABLE_V} {' ' * (70 - len(start_dt))}{start_dt} {_TABLE_V}"
+        )
+        lines.append(
+            f"{_TABLE_TM}{_TABLE_H * 70}{_TABLE_H * 0}"
+        )
+
+        lines.append(self._row("Category", "", "Status | Detail"))
+        lines.append(
+            f"{_TABLE_TML}{_TABLE_MH * 70}"
+        )
+
+        lines.append(self._row("", "", ""))
+
+        # ── Backup section ──
+        lines.append(
+            self._row(
+                "Backup Started",
+                "passed" if f["backup_started"] else "failed",
+                f["backup_started"] or "\u2014",
+            )
+        )
+        lines.append(
+            self._row(
+                "Backup Completed",
+                "passed" if f["backup_completed"] else "failed",
+                f["backup_completed"] or "\u2014",
+            )
+        )
+        lines.append(
+            self._row(
+                "Files Backed Up",
+                "passed" if f["backup_files"] > 0 else "failed",
+                str(f["backup_files"]),
+            )
+        )
+        lines.append(
+            self._row(
+                "Total Backup Size",
+                "passed" if f["backup_size"] > 0 else "failed",
+                _human_size(f["backup_size"]),
+            )
+        )
+
+        lines.append(self._sep())
+
+        # ── Deployment section ──
+        lines.append(
+            self._row(
+                "Commit Status",
+                f["commit_status"],
+                f["commit_hash"] or "\u2014",
+            )
+        )
+        lines.append(
+            self._row(
+                "Push Status",
+                f["push_status"],
+                _ok("Pushed") if f["push_status"] == "passed" else _fail("Failed"),
+            )
+        )
+        dep_icon = "passed" if f["deploy_status"] == "passed" else "failed"
+        lines.append(
+            self._row(
+                "HF Space Deployment",
+                dep_icon,
+                f["deploy_duration"] or "\u2014",
+            )
+        )
+
+        lines.append(self._sep())
+
+        # ── Restore section ──
+        lines.append(
+            self._row(
+                "Data Restoration",
+                f["restore_status"],
+                _ok("Restored") if f["restore_status"] == "passed" else _fail("Failed"),
+            )
+        )
+        lines.append(
+            self._row(
+                "Files Restored",
+                "passed" if f["restore_files"] > 0 else "failed",
+                str(f["restore_files"]),
+            )
+        )
+        lines.append(
+            self._row(
+                "Verification Status",
+                f["verification_status"],
+                _ok("Verified") if f["verification_status"] == "passed" else _fail("Failed"),
+            )
+        )
+
+        lines.append(self._sep())
+
+        # ── Summary section ──
+        lines.append(
+            self._row(
+                "Total Execution Time",
+                "passed",
+                f.get("total_duration", ""),
+            )
+        )
+        lines.append(
+            self._row(
+                "Errors",
+                "failed" if err_count > 0 else "passed",
+                str(err_count),
+            )
+        )
+        lines.append(
+            self._row(
+                "Warnings",
+                "passed" if warn_count == 0 else "skipped",
+                str(warn_count),
+            )
+        )
+
+        lines.append(
+            f"{_TABLE_BL}{_TABLE_H * 16}{_TABLE_H * 16}{_TABLE_H * 18}{_TABLE_H * 20}"
+            f"{_TABLE_BR}"
+        )
+
+        for line in lines:
+            print(line)
+
+        if err_count > 0:
+            print(f"\n{_fail('Errors:')}")
+            for e in f["errors"]:
+                print(f"  {_fail(_BULLET)} {e}")
+
+        if warn_count > 0:
+            print(f"\n{_warn('Warnings:')}")
+            for w in f["warnings"]:
+                print(f"  {_warn(_BULLET)} {w}")
+
+
+def run_deployment(args: argparse.Namespace, env: dict[str, str]) -> int:
+    audit = AuditReport()
+    cwd = os.getcwd()
+    persistence_dir = Path(cwd) / "persistence"
+    persistence_dir.mkdir(parents=True, exist_ok=True)
+
+    space_url = args.space_url or env.get("SPACE_URL", "https://aetherbase-llm-ready-data.hf.space")
+    api_key = args.api_key or env.get("api_key", env.get("API_KEY", ""))
+    hf_token = args.hf_token or env.get("hf_token", env.get("HF_TOKEN", ""))
+    skip_backup = args.skip_backup
+
+    if not api_key:
+        print(f"  {_fail('ERROR:')} No API key found. Set API_KEY in .env or pass --api-key.")
+        return 1
+
+    if not hf_token:
+        print(f"  {_warn('WARNING:')} No HF token. Can't verify Space build status. Set HF_TOKEN in .env or pass --hf-token.")
+        hf_token = None
+
+    auth_headers = {
+        "Authorization": f"Bearer {api_key}",
+        "User-Agent": "deploy.py/1.0",
+    }
+
+    remote_url = git_remote_url(cwd)
+    owner, space_name = _parse_hf_space_from_url(remote_url) if remote_url else ("aetherbase", "llm-ready-data")
+    hf_api_headers = None
+    if hf_token:
+        hf_api_headers = {
+            "Authorization": f"Bearer {hf_token}",
+            "User-Agent": "deploy.py/1.0",
+        }
+
+    # ──────────────────────────────────────────────
+    # PHASE 1: BACKUP (remote via API, fallback to local)
+    # ──────────────────────────────────────────────
+    print(f"\n{_bold('PHASE 1/4: Data Backup')}")
+    print(f"  {_INFO_ICON} Backing up data from: {space_url}")
+
+    if skip_backup:
+        print(f"  {_WARN_ICON} Backup skipped (--skip-backup)")
+        audit.set("backup_status", "skipped")
+    else:
+        backup_file = persistence_dir / BACKUP_ARCHIVE
+        backup_url = _build_url(space_url, BACKUP_ENDPOINT)
+        audit.set("backup_started", _timestamp())
+        backed_up = False
+
+        # Try remote backup endpoint first
+        try:
+            status, total_bytes = http_get_stream(backup_url, auth_headers, backup_file, timeout=120)
+            if status == 200:
+                backed_up = True
+                print(f"  {_OK_ICON} Remote backup successful")
+        except Exception as exc:
+            print(f"  {_WARN_ICON} Remote backup unavailable: {exc}")
+
+        # Fall back to local backup
+        if not backed_up:
+            data_dir = Path(cwd) / "data"
+            if data_dir.is_dir():
+                print(f"  {_INFO_ICON} Falling back to local backup: {data_dir}")
+                try:
+                    buf = io.BytesIO()
+                    with tarfile.open(fileobj=buf, mode="w:gz") as tar:
+                        for path in sorted(data_dir.rglob("*")):
+                            if path.is_file():
+                                arcname = path.relative_to(data_dir.parent)
+                                tar.add(str(path), arcname=str(arcname))
+                    buf.seek(0)
+                    backup_file.write_bytes(buf.read())
+
+                    total_bytes = backup_file.stat().st_size
+                    file_count = 0
+                    with tarfile.open(backup_file, "r:gz") as tar:
+                        file_count = sum(1 for m in tar.getmembers() if m.isfile())
+
+                    backed_up = True
+                    print(f"  {_OK_ICON} Local backup saved: {backup_file}")
+                except Exception as exc:
+                    msg = f"Local backup failed: {exc}"
+                    audit.error(msg)
+                    print(f"  {_FAIL_ICON} {msg}")
+                    audit.print()
+                    return 1
+            else:
+                msg = f"No data found locally ({data_dir}) and remote backup unavailable"
+                audit.warn(msg)
+                print(f"  {_WARN_ICON} {msg}")
+                audit.set("backup_files", 0)
+                audit.set("backup_size", 0)
+                audit.set("backup_completed", _timestamp())
+                audit.set("backup_status", "skipped")
+
+        if backed_up:
+            file_count = 0
+            try:
+                with tarfile.open(backup_file, "r:gz") as tar:
+                    file_count = sum(1 for m in tar.getmembers() if m.isfile())
+            except Exception:
+                audit.warn("Could not count files in backup archive")
+                file_count = backup_file.stat().st_size
+
+            audit.set("backup_completed", _timestamp())
+            audit.set("backup_files", file_count)
+            audit.set("backup_size", total_bytes)
+            audit.set("backup_status", "passed")
+            print(f"  {_OK_ICON} Files: {file_count} | Size: {_human_size(total_bytes)}")
+
+    # ──────────────────────────────────────────────
+    # PHASE 2: GIT COMMIT & PUSH
+    # ──────────────────────────────────────────────
+    print(f"\n{_bold('PHASE 2/4: Git Commit & Push')}")
+
+    changed = git_status(cwd)
+    if not changed:
+        print(f"  {_WARN_ICON} No changes to commit (working tree clean)")
+        audit.set("commit_status", "passed")
+        audit.set("commit_hash", "no-change")
+        audit.set("push_status", "passed")
+    else:
+        print(f"  {_INFO_ICON} Staging {len(changed)} file(s)...")
+        if not git_add_all(cwd):
+            msg = "Git add failed"
+            audit.error(msg)
+            audit.set("commit_status", "failed")
+            print(f"  {_FAIL_ICON} {msg}")
+            audit.print()
+            return 1
+
+        commit_msg = args.message or f"deploy: auto-deploy {_timestamp()}"
+        print(f"  {_INFO_ICON} Committing: {commit_msg}")
+        ok, sha = git_commit(cwd, commit_msg)
+        if not ok:
+            msg = f"Git commit failed: {sha}"
+            audit.error(msg)
+            audit.set("commit_status", "failed")
+            print(f"  {_FAIL_ICON} {msg}")
+            audit.print()
+            return 1
+
+        audit.set("commit_status", "passed")
+        audit.set("commit_hash", sha)
+        print(f"  {_OK_ICON} Committed: {sha}")
+
+        print(f"  {_INFO_ICON} Pushing to origin/main...")
+        ok, push_out = git_push(cwd)
+        if not ok:
+            msg = f"Git push failed"
+            audit.error(msg)
+            audit.set("push_status", "failed")
+            print(f"  {_FAIL_ICON} {msg}")
+            print(f"  {push_out}")
+            audit.print()
+            return 1
+
+        audit.set("push_status", "passed")
+        print(f"  {_OK_ICON} Push successful")
+
+    # ──────────────────────────────────────────────
+    # PHASE 3: WAIT FOR REDEPLOYMENT (via HF API)
+    # ──────────────────────────────────────────────
+    print(f"\n{_bold('PHASE 3/4: Waiting for HF Space Redeployment')}")
+
+    api_url = f"{HF_API_SPACES}/{owner}/{space_name}"
+    deploy_start = time.time()
+    max_wait = args.timeout
+    poll_interval = 10
+    waited = 0
+    deployed = False
+    last_error = ""
+    seen_building = False
+
+    print(f"  {_INFO_ICON} Tracking {owner}/{space_name} via HF API...")
+
+    if hf_api_headers:
+        initial = http_get_json(api_url, hf_api_headers, timeout=15) or {}
+        prev_stage = initial.get("runtime", {}).get("stage", "")
+
+        while waited < max_wait:
+            data = http_get_json(api_url, hf_api_headers, timeout=15)
+            stage = (data or {}).get("runtime", {}).get("stage", "")
+
+            if stage == "BUILDING":
+                if not seen_building:
+                    print(f"  {_INFO_ICON} Build started (stage: BUILDING)")
+                    seen_building = True
+
+            elif stage == "RUNNING":
+                if seen_building:
+                    print(f"  {_OK_ICON} Build complete (stage: RUNNING)")
+                    deployed = True
+                    break
+                if waited > 30 and prev_stage == "RUNNING":
+                    print(f"  {_OK_ICON} Space is running (may have skipped BUILDING stage)")
+                    deployed = True
+                    break
+
+            elif stage in ("PAUSED", "STOPPED", "NO_APP"):
+                last_error = f"Space in unexpected state: {stage}"
+                deployed = False
+                break
+
+            prev_stage = stage
+            time.sleep(poll_interval)
+            waited += poll_interval
+            if waited % 30 == 0 and not seen_building:
+                print(f"  {_INFO_ICON} Still waiting for build to start... ({waited}s)")
+            elif waited % 30 == 0 and seen_building:
+                print(f"  {_INFO_ICON} Still building... ({waited}s)")
+
+    deploy_duration = time.time() - deploy_start
+
+    if not hf_api_headers:
+        print(f"  {_WARN_ICON} No HF token — assuming deployment is underway")
+        print(f"  {_INFO_ICON} Waiting {max_wait}s for build + startup...")
+        time.sleep(max_wait)
+        deployed = True
+
+    if deployed:
+        health_url = _build_url(space_url, HEALTH_ENDPOINT)
+        try:
+            req = urllib.request.Request(health_url, method="GET")
+            with urllib.request.urlopen(req, timeout=15) as resp:
+                if resp.status == 200:
+                    print(f"  {_OK_ICON} App health check passed (new instance serving)")
+        except Exception:
+            print(f"  {_WARN_ICON} App health check unavailable (may still be starting)")
+
+        audit.set("deploy_status", "passed")
+        audit.set("deploy_duration", _human_duration(deploy_duration))
+        print(f"  {_OK_ICON} Space redeployed in {_human_duration(deploy_duration)}")
+    else:
+        audit.set("deploy_status", "failed")
+        audit.set("deploy_duration", _human_duration(deploy_duration))
+        msg = f"Space did not redeploy within {max_wait}s (last stage: {stage}, error: {last_error})"
+        audit.error(msg)
+        print(f"  {_FAIL_ICON} {msg}")
+        audit.print()
+        return 1
+
+    # ──────────────────────────────────────────────
+    # PHASE 4: DATA RESTORE
+    # ──────────────────────────────────────────────
+    print(f"\n{_bold('PHASE 4/4: Data Restoration')}")
+
+    if skip_backup:
+        print(f"  {_WARN_ICON} Restore skipped (no backup)")
+        audit.set("restore_status", "skipped")
+        audit.set("verification_status", "skipped")
+        audit.set("restore_files", 0)
+    else:
+        backup_file = persistence_dir / BACKUP_ARCHIVE
+        if not backup_file.is_file():
+            msg = f"Backup file not found: {backup_file}"
+            audit.error(msg)
+            audit.set("restore_status", "failed")
+            print(f"  {_FAIL_ICON} {msg}")
+            audit.print()
+            return 1
+
+        restore_url = _build_url(space_url, RESTORE_ENDPOINT)
+        print(f"  {_INFO_ICON} Restoring data to: {space_url}")
+
+        try:
+            status, body = http_post_multipart(
+                restore_url, backup_file, "file", auth_headers, timeout=300
+            )
+
+            if status == 200:
+                try:
+                    result = json.loads(body)
+                    restored = result.get("files_restored", 0)
+                except Exception:
+                    restored = 0
+
+                audit.set("restore_files", max(restored, 1))
+                audit.set("restore_status", "passed")
+                print(f"  {_OK_ICON} Data restored: {restored} files")
+
+                # Verification
+                print(f"  {_INFO_ICON} Verifying data restoration...")
+                try:
+                    verify_backup = persistence_dir / "verify_check.tar.gz"
+                    status_v, _ = http_get_stream(
+                        _build_url(space_url, BACKUP_ENDPOINT),
+                        auth_headers,
+                        verify_backup,
+                        timeout=120,
+                    )
+
+                    if status_v == 200:
+                        with tarfile.open(verify_backup, "r:gz") as tar:
+                            verify_files = sum(1 for m in tar.getmembers() if m.isfile())
+                        verify_backup.unlink(missing_ok=True)
+
+                        if verify_files >= audit.fields.get("backup_files", 0) * 0.9:
+                            audit.set("verification_status", "passed")
+                            print(f"  {_OK_ICON} Verification passed: {verify_files} files found")
+                        else:
+                            audit.set("verification_status", "failed")
+                            audit.warn(
+                                f"Verification file count ({verify_files}) mismatch with backup ({audit.fields.get('backup_files', 0)})"
+                            )
+                            print(f"  {_WARN_ICON} Verification: file count mismatch ({verify_files} vs {audit.fields.get('backup_files', 0)})")
+                    else:
+                        audit.set("verification_status", "failed")
+                        audit.warn(f"Verification request returned HTTP {status_v}")
+                        print(f"  {_WARN_ICON} Verification failed (HTTP {status_v})")
+                except Exception as exc:
+                    audit.set("verification_status", "failed")
+                    audit.warn(f"Verification error: {exc}")
+                    print(f"  {_WARN_ICON} Verification error: {exc}")
+            else:
+                audit.set("restore_status", "failed")
+                msg = f"Restore returned HTTP {status}: {body[:200]}"
+                audit.error(msg)
+                print(f"  {_FAIL_ICON} {msg}")
+                audit.print()
+                return 1
+
+        except Exception as exc:
+            audit.set("restore_status", "failed")
+            msg = f"Restore failed: {exc}"
+            audit.error(msg)
+            print(f"  {_FAIL_ICON} {msg}")
+            audit.print()
+            return 1
+
+    # ──────────────────────────────────────────────
+    # SUMMARY
+    # ──────────────────────────────────────────────
+    print(f"\n{_bold('=' * 70)}")
+    audit.print()
+
+    has_errors = len(audit.fields["errors"]) > 0
+    if has_errors:
+        print(f"\n{_fail('Deployment completed with errors.')}")
+        return 1
+    else:
+        print(f"\n{_ok('Deployment completed successfully.')}")
+        return 0
+
+
+# ─────────────────────────────────────────────────────────────
+# CLI entrypoint
+# ─────────────────────────────────────────────────────────────
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Deploy to Hugging Face Spaces with persistent data handling.",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog=(
+            "Environment variables (from .env or process env):\n"
+            "  API_KEY      API key for the Hugging Face Space\n"
+            "  SPACE_URL    Base URL of the deployed Space\n"
+            "  HF_TOKEN     Hugging Face API token (for build tracking)\n"
+        ),
+    )
+    parser.add_argument(
+        "--space-url",
+        default="",
+        help="Base URL of the Hugging Face Space (e.g. https://aetherbase-llm-ready-data.hf.space)",
+    )
+    parser.add_argument(
+        "--api-key",
+        default="",
+        help="API key for authenticating with the Space",
+    )
+    parser.add_argument(
+        "--message", "-m",
+        default="",
+        help="Git commit message",
+    )
+    parser.add_argument(
+        "--timeout",
+        type=int,
+        default=300,
+        help="Maximum wait time (seconds) for Space redeployment (default: 300)",
+    )
+    parser.add_argument(
+        "--hf-token",
+        default="",
+        help="Hugging Face API token (for tracking Space build status)",
+    )
+    parser.add_argument(
+        "--skip-backup",
+        action="store_true",
+        help="Skip the backup and restore phases",
+    )
+    parser.add_argument(
+        "--env-file",
+        default=".env",
+        help="Path to .env file (default: .env)",
+    )
+
+    args = parser.parse_args()
+    env = load_env(args.env_file)
+
+    # Merge with process environment (process env takes precedence)
+    for key in ("API_KEY", "SPACE_URL", "HF_TOKEN"):
+        if os.environ.get(key):
+            env[key] = os.environ[key]
+
+    try:
+        rc = run_deployment(args, env)
+    except KeyboardInterrupt:
+        print(f"\n{_warn('Deployment interrupted by user.')}")
+        rc = 130
+    except Exception as exc:
+        print(f"\n{_fail('Unexpected error: ' + str(exc))}")
+        rc = 1
+
+    sys.exit(rc)
+
+
+if __name__ == "__main__":
+    main()

tests/test_webhook_socket.py

@@ -0,0 +1,422 @@
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import hmac
+import json
+import os
+import sys
+import time
+from typing import AsyncGenerator
+
+import pytest
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
+from app.services.webhook_socket_service import ChannelManager, sign_payload, verify_signature
+
+API_KEY = "changeme"
+AUTH_HEADER = {"Authorization": f"Bearer {API_KEY}"}
+BASE = "http://localhost:7860/api/v1"
+
+
+# ---------------------------------------------------------------------------
+#  Unit tests for ChannelManager
+# ---------------------------------------------------------------------------
+
+class TestChannelManager:
+    def test_create_channel(self):
+        mgr = ChannelManager()
+        ch = mgr.create_channel()
+        assert ch.channel_id is not None
+        assert len(ch.channel_id) == 16
+        assert ch.buffer_size == 0
+        assert ch.secret is None
+
+    def test_create_channel_custom_id(self):
+        mgr = ChannelManager()
+        ch = mgr.create_channel(channel_id="my-channel")
+        assert ch.channel_id == "my-channel"
+
+    def test_create_channel_with_secret_and_buffer(self):
+        mgr = ChannelManager()
+        ch = mgr.create_channel(channel_id="secure", secret="s3cret", buffer_size=10)
+        assert ch.secret == "s3cret"
+        assert ch.buffer_size == 10
+
+    def test_get_channel(self):
+        mgr = ChannelManager()
+        mgr.create_channel(channel_id="abc")
+        assert mgr.get_channel("abc") is not None
+        assert mgr.get_channel("nonexistent") is None
+
+    def test_delete_channel(self):
+        mgr = ChannelManager()
+        mgr.create_channel(channel_id="del-me")
+        assert mgr.delete_channel("del-me") is True
+        assert mgr.get_channel("del-me") is None
+        assert mgr.delete_channel("del-me") is False
+
+    def test_create_duplicate_id(self):
+        mgr = ChannelManager()
+        mgr.create_channel(channel_id="dup")
+        ch2 = mgr.create_channel(channel_id="dup")
+        # Should overwrite
+        assert mgr.get_channel("dup") is ch2
+
+    def test_default_buffer_from_manager(self):
+        mgr = ChannelManager(default_buffer=25)
+        ch = mgr.create_channel()
+        assert ch.buffer_size == 25
+
+    def test_publish_nonexistent_channel(self):
+        mgr = ChannelManager()
+        result = asyncio.run(mgr.publish("no-such-channel", {"hello": "world"}))
+        assert result == -1
+
+    @pytest.mark.asyncio
+    async def test_publish_and_buffer(self):
+        mgr = ChannelManager()
+        ch = mgr.create_channel(channel_id="buf-test", buffer_size=3)
+
+        await mgr.publish("buf-test", {"n": 1})
+        await mgr.publish("buf-test", {"n": 2})
+        await mgr.publish("buf-test", {"n": 3})
+        await mgr.publish("buf-test", {"n": 4})
+
+        assert len(ch.history) == 3
+        assert ch.history[0]["payload"]["n"] == 2
+        assert ch.history[2]["payload"]["n"] == 4
+        assert ch.message_count == 4
+
+    def test_stats(self):
+        mgr = ChannelManager()
+        mgr.create_channel(channel_id="a")
+        mgr.create_channel(channel_id="b")
+        stats = mgr.stats()
+        assert stats["channels"] == 2
+        assert stats["total_subscribers"] == 0
+
+    def test_sign_and_verify(self):
+        secret = "my-secret"
+        body = b'{"hello":"world"}'
+        sig = sign_payload(secret, body)
+        assert sig.startswith("sha256=")
+        assert verify_signature(secret, body, sig) is True
+        assert verify_signature(secret, body, "sha256=bad") is False
+        assert verify_signature("wrong-secret", body, sig) is False
+
+    def test_sign_constant_result(self):
+        secret = "test"
+        body = b"data"
+        sig1 = sign_payload(secret, body)
+        sig2 = sign_payload(secret, body)
+        assert sig1 == sig2
+
+
+# ---------------------------------------------------------------------------
+#  Integration tests (requires running server @ localhost:7860)
+# ---------------------------------------------------------------------------
+
+pytestmark_integration = pytest.mark.skipif(
+    not os.environ.get("RUN_INTEGRATION_TESTS"),
+    reason="Set RUN_INTEGRATION_TESTS=1 to run integration tests",
+)
+
+
+def _sign(body: bytes, secret: str) -> str:
+    return "sha256=" + hmac.new(secret.encode(), body, hashlib.sha256).hexdigest()
+
+
+@pytest.mark.skipif(
+    not os.environ.get("RUN_INTEGRATION_TESTS"),
+    reason="Set RUN_INTEGRATION_TESTS=1 to run integration tests",
+)
+class TestWebhookSocketIntegration:
+    @pytest.fixture(autouse=True)
+    async def _setup(self):
+        import httpx
+
+        async with httpx.AsyncClient(base_url=BASE) as client:
+            self.client = client
+            yield
+
+    async def _create_channel(self, **kwargs) -> dict:
+        resp = await self.client.post("/channels", json=kwargs, headers=AUTH_HEADER)
+        assert resp.status_code == 201
+        return resp.json()
+
+    async def _delete_channel(self, channel_id: str):
+        resp = await self.client.delete(f"/channels/{channel_id}", headers=AUTH_HEADER)
+        return resp.status_code == 200
+
+    async def test_health(self):
+        resp = await self.client.get("/health")
+        assert resp.status_code == 200
+        data = resp.json()
+        assert data["success"] is True
+
+    async def test_create_and_list_channels(self):
+        ch = await self._create_channel(channel_id="test-list", buffer_size=5)
+        assert ch["channel_id"] == "test-list"
+
+        resp = await self.client.get("/channels", headers=AUTH_HEADER)
+        assert resp.status_code == 200
+        data = resp.json()
+        assert any(c["channel_id"] == "test-list" for c in data["channels"])
+
+        await self._delete_channel("test-list")
+
+    async def test_create_channel_no_auth(self):
+        resp = await self.client.post("/channels", json={})
+        assert resp.status_code == 403
+
+    async def test_create_duplicate_channel(self):
+        await self._create_channel(channel_id="dup-test")
+        resp = await self.client.post("/channels", json={"channel_id": "dup-test"}, headers=AUTH_HEADER)
+        assert resp.status_code == 409
+        await self._delete_channel("dup-test")
+
+    async def test_channel_info(self):
+        await self._create_channel(channel_id="info-test")
+        resp = await self.client.get("/channels/info-test", headers=AUTH_HEADER)
+        assert resp.status_code == 200
+        data = resp.json()
+        assert data["channel_id"] == "info-test"
+        assert data["subscribers"] == 0
+        assert data["messages"] == 0
+        await self._delete_channel("info-test")
+
+    async def test_channel_info_not_found(self):
+        resp = await self.client.get("/channels/no-such", headers=AUTH_HEADER)
+        assert resp.status_code == 404
+
+    async def test_delete_channel(self):
+        await self._create_channel(channel_id="delete-me")
+        resp = await self.client.delete("/channels/delete-me", headers=AUTH_HEADER)
+        assert resp.status_code == 200
+        assert resp.json()["deleted"] == "delete-me"
+
+        resp = await self.client.get("/channels/delete-me", headers=AUTH_HEADER)
+        assert resp.status_code == 404
+
+    async def test_delete_channel_not_found(self):
+        resp = await self.client.delete("/channels/no-such", headers=AUTH_HEADER)
+        assert resp.status_code == 404
+
+    async def test_webhook_delivers_to_ws(self):
+        ch = await self._create_channel(channel_id="ws-deliver", buffer_size=5)
+        cid = ch["channel_id"]
+
+        import websockets
+
+        ws_url = f"ws://localhost:7860/api/v1/ws/{cid}"
+
+        async with websockets.connect(ws_url) as ws:
+            connected = json.loads(await ws.recv())
+            assert connected["event"] == "connected"
+            assert connected["channel"] == cid
+
+            payload = {"msg": "hello from webhook", "num": 42}
+            resp = await self.client.post(f"/webhook/{cid}", json=payload)
+            assert resp.status_code == 200
+            wh_data = resp.json()
+            assert wh_data["status"] == "delivered"
+            assert wh_data["subscribers_notified"] == 1
+
+            received = json.loads(await ws.recv())
+            assert received["event"] == "message"
+            assert received["channel"] == cid
+            assert received["payload"] == payload
+
+        await self._delete_channel(cid)
+
+    async def test_webhook_no_subscribers(self):
+        ch = await self._create_channel(channel_id="no-subs")
+        resp = await self.client.post(f"/webhook/{ch['channel_id']}", json={"data": 1})
+        assert resp.status_code == 200
+        assert resp.json()["subscribers_notified"] == 0
+        await self._delete_channel(ch["channel_id"])
+
+    async def test_webhook_not_found(self):
+        resp = await self.client.post("/webhook/no-such", json={"x": 1})
+        assert resp.status_code == 404
+
+    async def test_hook_alias(self):
+        ch = await self._create_channel(channel_id="hook-alias")
+        resp = await self.client.post(f"/hook/{ch['channel_id']}", json={"test": True})
+        assert resp.status_code == 200
+        assert resp.json()["status"] == "delivered"
+        await self._delete_channel(ch["channel_id"])
+
+    async def test_hmac_signed_webhook(self):
+        secret = "hmac-test-secret"
+        ch = await self._create_channel(channel_id="hmac-test", secret=secret)
+        cid = ch["channel_id"]
+
+        payload = b'{"signed": "data"}'
+        sig = _sign(payload, secret)
+
+        resp = await self.client.post(
+            f"/webhook/{cid}",
+            content=payload,
+            headers={"Content-Type": "application/json", "X-Signature-256": sig},
+        )
+        assert resp.status_code == 200, resp.text
+
+        resp_bad = await self.client.post(
+            f"/webhook/{cid}",
+            content=payload,
+            headers={"Content-Type": "application/json", "X-Signature-256": "sha256=bad"},
+        )
+        assert resp_bad.status_code == 401
+
+        resp_no_sig = await self.client.post(
+            f"/webhook/{cid}",
+            content=payload,
+            headers={"Content-Type": "application/json"},
+        )
+        assert resp_no_sig.status_code == 401
+
+        await self._delete_channel(cid)
+
+    async def test_webhook_form_urlencoded(self):
+        ch = await self._create_channel(channel_id="form-test")
+        cid = ch["channel_id"]
+
+        resp = await self.client.post(
+            f"/webhook/{cid}",
+            data={"field1": "value1", "field2": "value2"},
+            headers={"Content-Type": "application/x-www-form-urlencoded"},
+        )
+        assert resp.status_code == 200
+        assert resp.json()["status"] == "delivered"
+
+        await self._delete_channel(cid)
+
+    async def test_webhook_raw_text(self):
+        ch = await self._create_channel(channel_id="raw-test")
+        cid = ch["channel_id"]
+
+        resp = await self.client.post(
+            f"/webhook/{cid}",
+            content="just some raw text",
+            headers={"Content-Type": "text/plain"},
+        )
+        assert resp.status_code == 200
+
+        await self._delete_channel(cid)
+
+    async def test_ws_replay_buffer(self):
+        ch = await self._create_channel(channel_id="replay-test", buffer_size=3)
+        cid = ch["channel_id"]
+
+        await self.client.post(f"/webhook/{cid}", json={"n": 1})
+        await self.client.post(f"/webhook/{cid}", json={"n": 2})
+        await self.client.post(f"/webhook/{cid}", json={"n": 3})
+
+        import websockets
+
+        ws_url = f"ws://localhost:7860/api/v1/ws/{cid}"
+        async with websockets.connect(ws_url) as ws:
+            connected = json.loads(await ws.recv())
+            assert connected["event"] == "connected"
+            assert connected["buffered"] == 3
+
+            for expected_n in [1, 2, 3]:
+                msg = json.loads(await ws.recv())
+                assert msg["payload"]["n"] == expected_n
+
+        await self._delete_channel(cid)
+
+    async def test_ws_auth_with_secret(self):
+        ch = await self._create_channel(channel_id="ws-auth-test", secret="topsecret")
+        cid = ch["channel_id"]
+
+        import websockets
+
+        ws_url = f"ws://localhost:7860/api/v1/ws/{cid}"
+
+        async with websockets.connect(f"{ws_url}?secret=topsecret") as ws:
+            msg = json.loads(await ws.recv())
+            assert msg["event"] == "connected"
+
+        async with websockets.connect(ws_url) as ws:
+            msg = json.loads(await ws.recv())
+            assert msg["event"] == "error"
+
+        await self._delete_channel(cid)
+
+    async def test_ws_channel_not_found(self):
+        import websockets
+
+        async with websockets.connect("ws://localhost:7860/api/v1/ws/does-not-exist") as ws:
+            msg = json.loads(await ws.recv())
+            assert msg["event"] == "error"
+            assert "channel not found" in msg["message"]
+
+    async def test_stats_endpoint(self):
+        resp = await self.client.get("/webhook-socket/stats", headers=AUTH_HEADER)
+        assert resp.status_code == 200
+        data = resp.json()
+        assert "channels" in data
+        assert "total_messages" in data
+        assert "total_subscribers" in data
+
+    async def test_full_lifecycle(self):
+        cid = "lifecycle-test"
+
+        ch = await self._create_channel(channel_id=cid, buffer_size=10, secret="life-secret")
+        assert ch["channel_id"] == cid
+
+        info_resp = await self.client.get(f"/channels/{cid}", headers=AUTH_HEADER)
+        assert info_resp.status_code == 200
+        assert info_resp.json()["messages"] == 0
+
+        import websockets
+
+        ws_url = f"ws://localhost:7860/api/v1/ws/{cid}?secret=life-secret"
+        async with websockets.connect(ws_url) as ws:
+            connected = json.loads(await ws.recv())
+            assert connected["event"] == "connected"
+
+            payload = {"event_type": "push", "data": {"ref": "main"}}
+            sig = _sign(json.dumps(payload).encode(), "life-secret")
+            resp = await self.client.post(
+                f"/webhook/{cid}",
+                json=payload,
+                headers={"X-Signature-256": sig, "X-GitHub-Event": "push"},
+            )
+            assert resp.status_code == 200
+            wh = resp.json()
+            assert wh["subscribers_notified"] == 1
+
+            received = json.loads(await ws.recv())
+            assert received["event"] == "message"
+            assert received["payload"]["event_type"] == "push"
+            assert received["headers"]["X-GitHub-Event"] == "push"
+
+        stats_resp = await self.client.get(f"/channels/{cid}", headers=AUTH_HEADER)
+        assert stats_resp.json()["messages"] == 1
+
+        await self._delete_channel(cid)
+
+        not_found = await self.client.get(f"/channels/{cid}", headers=AUTH_HEADER)
+        assert not_found.status_code == 404
+
+
+# ---------------------------------------------------------------------------
+#  Runner (standalone)
+# ---------------------------------------------------------------------------
+
+if __name__ == "__main__":
+    import subprocess
+    import sys as _sys
+
+    os.environ["RUN_INTEGRATION_TESTS"] = "1"
+    _sys.exit(
+        subprocess.run(
+            [_sys.executable, "-m", "pytest", __file__, "-v", "--tb=short"],
+            cwd=os.path.join(os.path.dirname(__file__), ".."),
+        ).returncode
+    )