scripts/sync_hf.py

#!/usr/bin/env python3
"""
Hermes Agent HF Spaces Persistence — Secure Atomic State Sync
================================================================

The production path uses the repository's strict backup allowlist, secret scan,
SHA-256 manifest, atomic Dataset commit, validated staging restore, and
last-known-good snapshot. Application source and runtime secrets are never
uploaded or activated from the Dataset.
"""

import os
import sys
import time
import json
import hashlib
import threading
import subprocess
import signal
import shutil
import secrets
import tempfile
import traceback
import fcntl
from contextlib import contextmanager
from pathlib import Path
from datetime import datetime, timezone
# Set timeout BEFORE importing huggingface_hub
os.environ.setdefault("HF_HUB_DOWNLOAD_TIMEOUT", "300")
os.environ.setdefault("HF_HUB_UPLOAD_TIMEOUT", "600")
os.environ.setdefault("HF_HUB_DISABLE_PROGRESS_BARS", "1")
os.environ.setdefault("HF_HUB_VERBOSITY", "warning")

import logging as _logging
_logging.getLogger("huggingface_hub").setLevel(_logging.WARNING)
_logging.getLogger("huggingface_hub.utils").setLevel(_logging.WARNING)
_logging.getLogger("filelock").setLevel(_logging.WARNING)

from huggingface_hub import HfApi
from runtime_policy import apply_optional_mcp_policy
from secure_backup import build_backup
from save_to_dataset_atomic import AtomicDatasetSaver
from restore_from_dataset_atomic import AtomicDatasetRestorer


def _replace_exact_once(code: str, old: str, new: str, *, label: str) -> tuple[str, bool]:
    """Apply a pinned-source patch only when the expected shape is unambiguous."""
    count = code.count(old)
    if count == 0:
        return code, False
    if count != 1:
        raise RuntimeError(f"{label}: expected exactly one match, found {count}")
    return code.replace(old, new, 1), True


def _replace_required_once(code: str, old: str, new: str, *, label: str) -> str:
    """Apply one critical patch or fail instead of silently degrading startup."""
    updated, changed = _replace_exact_once(code, old, new, label=label)
    if not changed:
        raise RuntimeError(f"{label}: required upstream marker was not found")
    return updated


# ── Logging helper ──────────────────────────────────────────────────────────

class TeeLogger:
    """Duplicate output to stream and file."""
    def __init__(self, filename, stream):
        self.stream = stream
        self.file = open(filename, "a", encoding="utf-8")
    def write(self, message):
        self.stream.write(message)
        self.file.write(message)
        self.flush()
    def flush(self):
        self.stream.flush()
        self.file.flush()
    def fileno(self):
        return self.stream.fileno()


# ── Analysis model resolution ───────────────────────────────────────────────

DEFAULT_OPENROUTER_ANALYSIS_MODEL = "openai/gpt-oss-20b:free"
DEFAULT_GOOGLE_ANALYSIS_MODEL = "gemini-2.5-flash"
DEFAULT_HF_ANALYSIS_MODEL = "meta-llama/Llama-3.1-8B-Instruct"


def _first_nonempty_env(*keys: str) -> tuple[str, str]:
    """Return (value, env_key) for the first non-empty variable."""
    for key in keys:
        value = os.environ.get(key, "").strip()
        if value:
            return value, key
    return "", ""


def resolve_openrouter_analysis_model(*, hermes_key: str) -> tuple[str, str]:
    """Resolve OpenRouter model for fallback/auxiliary chains."""
    value, source = _first_nonempty_env(hermes_key, "OPENROUTER_ANALYSIS_MODEL")
    if value:
        return validate_openrouter_free_model(value), source
    return DEFAULT_OPENROUTER_ANALYSIS_MODEL, "default"


def resolve_google_analysis_model(*, hermes_key: str) -> tuple[str, str]:
    """Resolve Gemini model for fallback/auxiliary chains."""
    value, source = _first_nonempty_env(hermes_key, "GOOGLE_ANALYSIS_MODEL")
    if value:
        return value, source
    return DEFAULT_GOOGLE_ANALYSIS_MODEL, "default"


def resolve_hf_analysis_model() -> tuple[str, str]:
    """Resolve Hugging Face model for the fallback chain."""
    value, source = _first_nonempty_env("HERMES_HF_FALLBACK_MODEL", "HF_ANALYSIS_MODEL")
    if value:
        return value, source
    return DEFAULT_HF_ANALYSIS_MODEL, "default"


def _log_analysis_model(provider: str, model: str, source: str, *, env_hint: str) -> None:
    if source == "default":
        print(
            f"[SYNC] {provider} analysis model: {model} "
            f"(default; set {env_hint} to override)"
        )
    else:
        print(f"[SYNC] {provider} analysis model: {model} (from {source})")


FUTURES_ADVISORY_TASK = "futures_advisory"
_OBSOLETE_ADVISORY_MODEL_MARKERS = ("openrouter/free",)


def _openrouter_model_requires_refresh(model: str) -> bool:
    """Paid OpenRouter models (no ``:free`` suffix) must be replaced."""
    return ":free" not in str(model or "").strip().lower()


def validate_openrouter_free_model(model: str) -> str:
    """Refuse paid OpenRouter models; fall back to the default free route."""
    cleaned = str(model or "").strip()
    if not cleaned:
        return DEFAULT_OPENROUTER_ANALYSIS_MODEL
    if _openrouter_model_requires_refresh(cleaned):
        print(
            f"[SYNC] WARNING: OpenRouter analysis model {cleaned!r} lacks :free suffix; "
            f"refusing paid usage and using {DEFAULT_OPENROUTER_ANALYSIS_MODEL}"
        )
        return DEFAULT_OPENROUTER_ANALYSIS_MODEL
    return cleaned


def _futures_advisory_aux_timeout() -> int:
    """Hermes per-task timeout for futures_advisory (overlay still caps total)."""
    try:
        return max(1, min(60, int(float(os.environ.get("EXTERNAL_AI_TOTAL_TIMEOUT_SECONDS", "15")))))
    except (TypeError, ValueError):
        return 15


def build_advisory_fallback_chain() -> list[dict[str, str]]:
    """Build credential-aware OpenRouter → Gemini → HuggingFace fallback chain."""
    chain: list[dict[str, str]] = []
    if os.environ.get("OPENROUTER_API_KEY", "").strip():
        openrouter_model, or_source = resolve_openrouter_analysis_model(
            hermes_key="HERMES_OPENROUTER_FALLBACK_MODEL"
        )
        _log_analysis_model(
            "OpenRouter",
            openrouter_model,
            or_source,
            env_hint="OPENROUTER_ANALYSIS_MODEL or HERMES_OPENROUTER_FALLBACK_MODEL",
        )
        chain.append({"provider": "openrouter", "model": openrouter_model})
    if os.environ.get("GOOGLE_API_KEY", "").strip():
        google_model, gm_source = resolve_google_analysis_model(
            hermes_key="HERMES_GOOGLE_FALLBACK_MODEL"
        )
        _log_analysis_model(
            "Gemini",
            google_model,
            gm_source,
            env_hint="GOOGLE_ANALYSIS_MODEL or HERMES_GOOGLE_FALLBACK_MODEL",
        )
        chain.append({"provider": "gemini", "model": google_model})
    if os.environ.get("HF_TOKEN", "").strip():
        hf_model, hf_source = resolve_hf_analysis_model()
        if hf_source == "default":
            print(
                f"[SYNC] HuggingFace analysis model: {hf_model} "
                "(default; set HF_ANALYSIS_MODEL or HERMES_HF_FALLBACK_MODEL to override)"
            )
        else:
            print(f"[SYNC] HuggingFace analysis model: {hf_model} (from {hf_source})")
        chain.append({"provider": "huggingface", "model": hf_model})
    return chain


def _chain_provider_models(chain: object) -> list[tuple[str, str]]:
    if not isinstance(chain, list):
        return []
    pairs: list[tuple[str, str]] = []
    for entry in chain:
        if not isinstance(entry, dict):
            continue
        provider = str(entry.get("provider") or "").strip().lower()
        model = str(entry.get("model") or "").strip()
        if provider and model:
            pairs.append((provider, model))
    return pairs


def advisory_chain_labels(chain: list[dict[str, str]]) -> str:
    return " -> ".join(f"{item['provider']}:{item['model']}" for item in chain)


def is_obsolete_advisory_chain(chain: object) -> bool:
    """True when the chain is missing, weak, or uses retired free-tier routes."""
    pairs = _chain_provider_models(chain)
    if not pairs:
        return True
    for _provider, model in pairs:
        model_lower = model.lower()
        if any(marker in model_lower for marker in _OBSOLETE_ADVISORY_MODEL_MARKERS):
            return True
        if _provider == "openrouter" and _openrouter_model_requires_refresh(model):
            return True
    return False


def futures_advisory_auxiliary_config(chain: list[dict[str, str]]) -> dict[str, object]:
    """Hermes-native auxiliary.futures_advisory block for advisory routing."""
    return {
        "provider": "auto",
        "model": "",
        "timeout": _futures_advisory_aux_timeout(),
        "fallback_chain": [dict(entry) for entry in chain],
    }


def advisory_config_is_current(
    fallback_providers: object,
    auxiliary_block: object,
    desired_chain: list[dict[str, str]],
) -> bool:
    """True when persisted Hermes config already matches the desired advisory chain."""
    if is_obsolete_advisory_chain(fallback_providers):
        return False
    if _chain_provider_models(fallback_providers) != _chain_provider_models(desired_chain):
        return False
    if not isinstance(auxiliary_block, dict):
        return False
    aux_chain = auxiliary_block.get("fallback_chain")
    if _chain_provider_models(aux_chain) != _chain_provider_models(desired_chain):
        return False
    if str(auxiliary_block.get("provider", "auto")).strip().lower() != "auto":
        return False
    return True


# ── Configuration ───────────────────────────────────────────────────────────

HF_TOKEN      = os.environ.get("HF_TOKEN")
HERMES_DATA   = Path("/opt/data")
APP_DIR       = Path("/opt/hermes")
DATASET_PATH  = "hermes_data"

AGENT_NAME = os.environ.get("AGENT_NAME", "HermesFace")

# HF Spaces built-in env vars (auto-set by HF runtime)
SPACE_HOST = os.environ.get("SPACE_HOST", "")
SPACE_ID   = os.environ.get("SPACE_ID", "")

SYNC_INTERVAL = int(os.environ.get("SYNC_INTERVAL", "60"))
AUTO_CREATE_DATASET = os.environ.get("AUTO_CREATE_DATASET", "true").lower() in ("true", "1", "yes")

# Dataset repo: auto-derive from SPACE_ID when not explicitly set.
# Format: {username}/{SpaceName}-data
HF_REPO_ID = os.environ.get("HERMES_DATASET_REPO", "")
if not HF_REPO_ID and SPACE_ID:
    HF_REPO_ID = f"{SPACE_ID}-data"
    print(f"[SYNC] HERMES_DATASET_REPO not set — auto-derived from SPACE_ID: {HF_REPO_ID}")
elif not HF_REPO_ID and HF_TOKEN:
    try:
        _api = HfApi(token=HF_TOKEN)
        _username = _api.whoami()["name"]
        HF_REPO_ID = f"{_username}/HermesFace-data"
        print(f"[SYNC] HERMES_DATASET_REPO not set — auto-derived from HF_TOKEN: {HF_REPO_ID}")
        del _api, _username
    except Exception as e:
        print(f"[SYNC] WARNING: Could not derive username from HF_TOKEN: {e}")
        HF_REPO_ID = ""

# Setup logging
log_dir = HERMES_DATA / "logs"
log_dir.mkdir(parents=True, exist_ok=True)
sys.stdout = TeeLogger(log_dir / "sync.log", sys.stdout)
sys.stderr = sys.stdout


# ── Sync Manager ────────────────────────────────────────────────────────────

class HermesFullSync:
    """Securely synchronize allowlisted durable state with an HF Dataset."""

    def __init__(self):
        self.enabled = False
        self.dataset_exists = False
        self.api = None
        self._sync_lock = threading.Lock()
        self._sync_lock_path = HERMES_DATA / ".sync.lock"

        if not HF_TOKEN:
            print("[SYNC] WARNING: HF_TOKEN not set. Persistence disabled.")
            return
        if not HF_REPO_ID:
            print("[SYNC] WARNING: Could not determine dataset repo (no SPACE_ID or HERMES_DATASET_REPO).")
            print("[SYNC] Persistence disabled.")
            return

        self.enabled = True
        self.api = HfApi(token=HF_TOKEN)
        self.dataset_exists = self._ensure_repo_exists()
        # Tracks the last set of secret-scan-excluded paths we already
        # announced, so a static, never-changing set of vendored/bundled
        # skill files isn't re-logged as a fresh WARNING on every 60s cycle.
        self._last_reported_rejected_paths: Optional[frozenset] = None

    # ── Repo management ────────────────────────────────────────────────

    def _ensure_repo_exists(self):
        """Check if dataset repo exists; auto-create only when AUTO_CREATE_DATASET=true."""
        try:
            self.api.repo_info(repo_id=HF_REPO_ID, repo_type="dataset")
            print(f"[SYNC] Dataset repo found: {HF_REPO_ID}")
            return True
        except Exception:
            if not AUTO_CREATE_DATASET:
                print(f"[SYNC] Dataset repo NOT found: {HF_REPO_ID}")
                print("[SYNC]   Set AUTO_CREATE_DATASET=true to auto-create.")
                print("[SYNC] Persistence disabled (app will still run normally).")
                return False
            print(f"[SYNC] Dataset repo NOT found: {HF_REPO_ID} — creating...")
            try:
                self.api.create_repo(
                    repo_id=HF_REPO_ID,
                    repo_type="dataset",
                    private=True,
                )
                print(f"[SYNC] Dataset repo created: {HF_REPO_ID}")
                return True
            except Exception as e:
                print(f"[SYNC] Failed to create dataset repo: {e}")
                return False

    # ── Secure synchronization ─────────────────────────────────────────

    @contextmanager
    def _exclusive_sync(self, operation: str):
        """Prevent overlapping thread or process sync operations."""
        if not self._sync_lock.acquire(blocking=False):
            print(f"[SYNC] {operation} skipped: another sync operation is active")
            yield False
            return
        lock_handle = None
        acquired_file_lock = False
        try:
            HERMES_DATA.mkdir(parents=True, exist_ok=True)
            lock_handle = self._sync_lock_path.open("a+")
            try:
                fcntl.flock(lock_handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
                acquired_file_lock = True
            except BlockingIOError:
                print(f"[SYNC] {operation} skipped: another sync process is active")
                yield False
                return
            yield True
        finally:
            if lock_handle is not None:
                if acquired_file_lock:
                    try:
                        fcntl.flock(lock_handle.fileno(), fcntl.LOCK_UN)
                    except OSError:
                        pass
                lock_handle.close()
            self._sync_lock.release()

    @staticmethod
    def _assert_restore_target_safe() -> None:
        target = HERMES_DATA.resolve()
        source = APP_DIR.resolve()
        if target == source or target in source.parents or source in target.parents:
            raise RuntimeError(
                f"Refusing persistence target that overlaps application source: {target}"
            )

    # ── Restore (startup) ─────────────────────────────────────────────

    def load_from_repo(self):
        """Restore one fully validated secure snapshot into allowlisted state roots."""
        if not self.enabled:
            print("[SYNC] Persistence disabled - skipping restore")
            self._ensure_default_config()
            return
        if not self.dataset_exists:
            print(f"[SYNC] Dataset {HF_REPO_ID} does not exist - starting fresh")
            self._ensure_default_config()
            return

        self._assert_restore_target_safe()
        with self._exclusive_sync("restore") as acquired:
            if not acquired:
                self._ensure_default_config()
                return
            print(f"[SYNC] Restoring validated state from dataset {HF_REPO_ID} ...")
            restorer = AtomicDatasetRestorer(
                HF_REPO_ID, DATASET_PATH, api=self.api, token=HF_TOKEN
            )
            result = restorer.restore_secure_latest(
                HERMES_DATA,
                last_known_good_path=HERMES_DATA.parent / ".hermes-last-known-good.tar.gz",
                protected_paths=(APP_DIR,),
            )
            if result.get("success"):
                print(
                    "[SYNC] Secure restore completed "
                    f"commit={result.get('commit_sha')} sha256={result.get('archive_sha256')}"
                )
            elif result.get("no_prior_backup"):
                print(
                    "[SYNC] No prior backup found in dataset yet - starting fresh "
                    "(this is normal on first boot or after a fresh dataset repo)"
                )
            else:
                print(
                    "[SYNC] Secure restore rejected; existing state left unchanged: "
                    f"{result.get('error', 'unknown error')}"
                )

        self._ensure_default_config()
        self._debug_list_files()

    # ── Save (periodic + shutdown) ─────────────────────────────────────

    def save_to_repo(self):
        """Build, validate, and atomically commit an allowlisted secure snapshot."""
        if not self.enabled:
            return
        if not HERMES_DATA.exists():
            print("[SYNC] /opt/data does not exist, nothing to save.")
            return

        self._assert_restore_target_safe()
        with self._exclusive_sync("save") as acquired:
            if not acquired:
                return
            if not self._ensure_repo_exists():
                print(f"[SYNC] Dataset {HF_REPO_ID} unavailable - skipping save")
                return

            try:
                with tempfile.TemporaryDirectory(prefix="hermes-sync-") as tmpdir:
                    archive = Path(tmpdir) / "secure_state.tar.gz"
                    manifest = build_backup(HERMES_DATA, archive)
                    saver = AtomicDatasetSaver(
                        HF_REPO_ID, DATASET_PATH, api=self.api, token=HF_TOKEN
                    )
                    result = saver.save_secure_archive_atomic(
                        archive,
                        {
                            "environment": os.environ.get("HERMES_ENV", "production"),
                            "platform": "huggingface-spaces",
                            "app": "hermesface",
                            "space_id": SPACE_ID,
                            "created_at": datetime.now(timezone.utc).isoformat(),
                        },
                    )
                print(
                    "[SYNC] Secure upload completed "
                    f"commit={result.get('commit_id')} files={result.get('files_count')} "
                    f"sha256={manifest.get('archive_sha256')}"
                )
                rejected_files = manifest.get("rejected_files") or []
                rejected_path_set = frozenset(item["path"] for item in rejected_files)
                if rejected_files and rejected_path_set != self._last_reported_rejected_paths:
                    # Only announce this as a WARNING the first time we see it,
                    # or when the excluded set actually changes. A static list
                    # of bundled/vendored skill docs that always trip the
                    # secret-content scanner would otherwise re-log identically
                    # every SYNC_INTERVAL forever, even though nothing changed
                    # and the exclusion is working exactly as intended.
                    paths = ", ".join(sorted(rejected_path_set))
                    print(
                        f"[SYNC] WARNING: {len(rejected_files)} file(s) excluded from backup "
                        f"(secret-like content, not uploaded): {paths}"
                    )
                elif rejected_files:
                    print(
                        f"[SYNC] {len(rejected_files)} file(s) still excluded from backup "
                        "(unchanged from previous cycle; secret-like content)"
                    )
                self._last_reported_rejected_paths = rejected_path_set
            except Exception as e:
                print(f"[SYNC] Secure upload rejected: {e}")
                traceback.print_exc()

    # ── Config helpers ─────────────────────────────────────────────────

    def _ensure_default_config(self):
        """Ensure Hermes has config.yaml and .env for HF Spaces."""
        config_path = HERMES_DATA / "config.yaml"
        env_path = HERMES_DATA / ".env"
        soul_path = HERMES_DATA / "SOUL.md"

        # Bootstrap from Hermes templates if available
        if not config_path.exists():
            template = APP_DIR / "cli-config.yaml.example"
            if template.exists():
                shutil.copy2(str(template), str(config_path))
                print("[SYNC] Created config.yaml from Hermes template")
            else:
                # Minimal fallback config
                import yaml
                config = {
                    "agent": {"name": AGENT_NAME},
                    "server": {"host": "0.0.0.0", "port": 7860},
                }
                with open(config_path, "w") as f:
                    yaml.dump(config, f, default_flow_style=False)
                print(f"[SYNC] Created minimal config.yaml (agent={AGENT_NAME}, port=7860)")

        if not env_path.exists():
            template = APP_DIR / ".env.example"
            if template.exists():
                shutil.copy2(str(template), str(env_path))
                print("[SYNC] Created .env from Hermes template")
            else:
                env_lines = []
                for key in [
                    "OPENROUTER_API_KEY", "OPENAI_API_KEY", "ANTHROPIC_API_KEY",
                    "NOUS_API_KEY", "GOOGLE_API_KEY", "MISTRAL_API_KEY",
                    "TELEGRAM_BOT_TOKEN", "DISCORD_BOT_TOKEN", "SLACK_BOT_TOKEN",
                ]:
                    val = os.environ.get(key, "")
                    if val:
                        env_lines.append(f"{key}={val}")
                if env_lines:
                    with open(env_path, "w") as f:
                        f.write("\n".join(env_lines) + "\n")
                    print(f"[SYNC] Created .env with {len(env_lines)} keys")

        if not soul_path.exists():
            template = APP_DIR / "docker" / "SOUL.md"
            if template.exists():
                shutil.copy2(str(template), str(soul_path))
                print("[SYNC] Created SOUL.md from Hermes template")
            else:
                with open(soul_path, "w") as f:
                    f.write(f"# {AGENT_NAME}\n\nI am {AGENT_NAME}, a self-improving AI assistant powered by Hermes Agent.\n")
                print("[SYNC] Created default SOUL.md")

        self._ensure_fallback_chain()
        self._normalize_runtime_config()
        self._ensure_web_backend_configured()

    def _ensure_web_backend_configured(self):
        """Give web_search/web_extract a working backend with no API key.

        check_web_api_key() gates both tools and returns False whenever no
        paid provider (Firecrawl, Tavily, Brave, Exa, ...) has credentials
        configured - which is the default on a fresh Space. Hermes already
        ships a free, keyless DuckDuckGo backend plugin (`ddgs`, confirmed
        registered at boot). Pointing web.backend at it directly avoids the
        per-turn "tools will be unavailable" warning without requiring the
        operator to obtain any paid API key.
        """
        config_path = HERMES_DATA / "config.yaml"
        try:
            import yaml
            data = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {}
            web_cfg = data.setdefault("web", {})
            if not isinstance(web_cfg, dict):
                web_cfg = {}
                data["web"] = web_cfg
            current = str(web_cfg.get("backend") or "").strip().lower()
            has_any_paid_key = any(
                os.environ.get(key, "").strip()
                for key in (
                    "FIRECRAWL_API_KEY", "TAVILY_API_KEY", "BRAVE_API_KEY", "EXA_API_KEY",
                )
            )
            if not current and not has_any_paid_key:
                web_cfg["backend"] = os.environ.get("HERMES_DEFAULT_WEB_BACKEND", "ddgs").strip() or "ddgs"
                config_path.write_text(yaml.safe_dump(data, sort_keys=False), encoding="utf-8")
                print(f"[SYNC] web.backend defaulted to '{web_cfg['backend']}' (no paid web API key configured)")
        except Exception as e:
            print(f"[SYNC] WARNING: could not configure default web backend: {e}")

    def _debug_list_files(self):
        try:
            count = sum(1 for _, _, files in os.walk(HERMES_DATA) for _ in files)
            print(f"[SYNC] Local /opt/data: {count} files")
        except Exception as e:
            print(f"[SYNC] listing failed: {e}")

    # ── Admin auth wiring ───────────────────────────────────────────────
    def _configure_admin_auth(self):
        """Map HERMES_ADMIN_PASSWORD onto Hermes' real dashboard auth provider.

        Hermes' bundled password provider (plugins/dashboard_auth/basic) reads
        HERMES_DASHBOARD_BASIC_AUTH_USERNAME / _PASSWORD / _SECRET — nothing in
        Hermes reads HERMES_ADMIN_PASSWORD. Previously this Secret was set but
        unused, and since Hermes now requires an auth provider on any
        non-loopback bind (HF Spaces binds 0.0.0.0), the dashboard was failing
        closed (401 on every route) with no way to log in. This wires the
        existing secret into the mechanism Hermes actually checks.
        """
        admin_password = os.environ.get("HERMES_ADMIN_PASSWORD", "").strip()
        if not admin_password:
            print("[SYNC] WARNING: HERMES_ADMIN_PASSWORD not set — dashboard "
                  "login will be unavailable (non-loopback bind requires an "
                  "auth provider; the basic-auth plugin won't register without "
                  "a username+password).")
            return

        # Keep the runtime provider username identical to the username written
        # by entrypoint.sh.  Leaving this at the historical default (admin)
        # made a non-admin HERMES_ADMIN_USERNAME login fail even though the
        # config file contained the correct username.
        admin_username = os.environ.get("HERMES_ADMIN_USERNAME", "admin").strip() or "admin"
        os.environ.setdefault("HERMES_DASHBOARD_BASIC_AUTH_USERNAME", admin_username)
        os.environ["HERMES_DASHBOARD_BASIC_AUTH_PASSWORD"] = admin_password

        # Persist a stable HMAC signing secret across restarts so logged-in
        # sessions survive a Space restart instead of all being invalidated.
        if "HERMES_DASHBOARD_BASIC_AUTH_SECRET" not in os.environ:
            secret_path = HERMES_DATA / ".dashboard_auth_secret"
            try:
                if secret_path.exists():
                    secret = secret_path.read_text().strip()
                else:
                    secret = secrets.token_hex(32)
                    secret_path.parent.mkdir(parents=True, exist_ok=True)
                    secret_path.write_text(secret)
                    try:
                        os.chmod(secret_path, 0o600)
                    except Exception:
                        pass
                if secret:
                    os.environ["HERMES_DASHBOARD_BASIC_AUTH_SECRET"] = secret
            except Exception as e:
                print(f"[SYNC] WARNING: could not persist dashboard auth secret: {e}")

        print("[SYNC] Admin login wired: HERMES_DASHBOARD_BASIC_AUTH_USERNAME/PASSWORD "
              "set from HERMES_ADMIN_USERNAME/HERMES_ADMIN_PASSWORD")

    def _configure_audit_signing_secret(self):
        """Persist and wire HERMES_AUDIT_SIGNING_SECRET for the trading overlay.

        trading.domain.audit_repo requires this env var to write tamper-evident
        (HMAC-signed, hash-chained) audit rows. Without it every trade-cycle
        write falls back to an unsigned "legacy" row and logs a WARNING -
        every ~15s in practice. HERMES_ENV isn't "production" here so it never
        blocks anything, it just silently degrades audit integrity forever.
        Generate a secret once and persist it exactly like the dashboard auth
        secret, so it's stable across restarts. An operator-provided
        HERMES_AUDIT_SIGNING_SECRET Space secret always wins.
        """
        if os.environ.get("HERMES_AUDIT_SIGNING_SECRET", "").strip():
            return
        secret_path = HERMES_DATA / ".audit_signing_secret"
        try:
            if secret_path.exists():
                secret = secret_path.read_text().strip()
            else:
                secret = secrets.token_hex(32)  # 64 hex chars, well over the 32-char minimum
                secret_path.parent.mkdir(parents=True, exist_ok=True)
                secret_path.write_text(secret)
                try:
                    os.chmod(secret_path, 0o600)
                except Exception:
                    pass
            if secret:
                os.environ["HERMES_AUDIT_SIGNING_SECRET"] = secret
                print("[SYNC] Audit integrity signing configured (persisted secret)")
        except Exception as e:
            print(f"[SYNC] WARNING: could not configure audit signing secret: {e}")

    def _normalize_runtime_config(self):
        """Normalize resource and auxiliary settings for a free HF Space.

        This removes the empty ``max_concurrent_sessions`` warning and gives
        Hermes auxiliary text tasks an explicit 402-aware fallback ladder:
        OpenRouter free router -> Google AI Studio -> the main agent model.
        Existing explicit operator settings are preserved unless they are
        empty/``auto`` and HERMES_MANAGE_AUXILIARY_FALLBACKS is enabled.
        """
        config_path = HERMES_DATA / "config.yaml"
        try:
            import yaml
            data = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {}
            changed = False

            current_limit = data.get("max_concurrent_sessions")
            if current_limit is None or current_limit == {} or current_limit == "":
                try:
                    limit = max(0, int(os.environ.get("HERMES_MAX_CONCURRENT_SESSIONS", "1")))
                except (TypeError, ValueError):
                    limit = 1
                data["max_concurrent_sessions"] = limit
                changed = True

            manage_aux = os.environ.get(
                "HERMES_MANAGE_AUXILIARY_FALLBACKS", "true"
            ).strip().lower() in {"1", "true", "yes", "on"}
            # Auxiliary tasks default to "openrouter"/"nous" in Hermes' own
            # upstream template even when no credential for either exists on
            # this Space. Since neither is authenticated here, every
            # compression/title_generation/web_extract call was probing both,
            # failing both ("payment / credit error", "no Nous authentication
            # found"), and only then falling through - one noisy WARNING pair
            # per aux call, forever. Treat a configured provider with no
            # working credential the same as "auto" unless the operator has
            # explicitly opted to keep it (HERMES_PRESERVE_AUXILIARY_PROVIDERS).
            preserve_aux_providers = os.environ.get(
                "HERMES_PRESERVE_AUXILIARY_PROVIDERS", "false"
            ).strip().lower() in {"1", "true", "yes", "on"}
            has_openrouter_key = bool(os.environ.get("OPENROUTER_API_KEY", "").strip())
            has_nous_auth = (HERMES_DATA / "auth.json").is_file()
            if manage_aux:
                auxiliary = data.setdefault("auxiliary", {})
                openrouter_model, or_source = resolve_openrouter_analysis_model(
                    hermes_key="HERMES_AUX_OPENROUTER_MODEL"
                )
                google_model, gm_source = resolve_google_analysis_model(
                    hermes_key="HERMES_AUX_GOOGLE_MODEL"
                )
                _log_analysis_model(
                    "OpenRouter",
                    openrouter_model,
                    or_source,
                    env_hint="OPENROUTER_ANALYSIS_MODEL or HERMES_AUX_OPENROUTER_MODEL",
                )
                _log_analysis_model(
                    "Gemini",
                    google_model,
                    gm_source,
                    env_hint="GOOGLE_ANALYSIS_MODEL or HERMES_AUX_GOOGLE_MODEL",
                )
                for task in ("compression", "title_generation", "web_extract"):
                    task_cfg = auxiliary.setdefault(task, {})
                    if not isinstance(task_cfg, dict):
                        task_cfg = {}
                        auxiliary[task] = task_cfg
                    configured_provider = str(task_cfg.get("provider", "auto")).strip().lower()
                    is_unusable_default = (
                        not preserve_aux_providers
                        and (
                            (configured_provider == "openrouter" and not has_openrouter_key)
                            or (configured_provider == "nous" and not has_nous_auth)
                        )
                    )
                    if configured_provider in {"", "auto"} or is_unusable_default:
                        if os.environ.get("OPENROUTER_API_KEY", "").strip():
                            task_cfg["provider"] = "openrouter"
                            task_cfg["model"] = openrouter_model
                            fallback_chain = []
                            if os.environ.get("GOOGLE_API_KEY", "").strip():
                                fallback_chain.append({"provider": "gemini", "model": google_model})
                            fallback_chain.append({"provider": "main"})
                            task_cfg["fallback_chain"] = fallback_chain
                        elif os.environ.get("GOOGLE_API_KEY", "").strip():
                            task_cfg["provider"] = "gemini"
                            task_cfg["model"] = google_model
                            task_cfg["fallback_chain"] = [{"provider": "main"}]
                        else:
                            task_cfg["provider"] = "main"
                            task_cfg.pop("model", None)
                            task_cfg.pop("fallback_chain", None)
                        changed = True

            if changed:
                config_path.write_text(
                    yaml.safe_dump(data, sort_keys=False), encoding="utf-8"
                )
                print("[SYNC] Runtime config normalized: session cap and auxiliary fallback chain ready")
        except Exception as e:
            print(f"[SYNC] WARNING: could not normalize runtime config: {e}")

    # ── Provider fallback chain ─────────────────────────────────────────
    def _ensure_fallback_chain(self):
        """Persist Hermes-native advisory provider config when needed.

        Writes top-level ``fallback_providers`` and ``auxiliary.futures_advisory``
        from credential-aware env resolution. Once the chain is current, set
        ``HERMES_PRESERVE_FALLBACK_PROVIDERS=true`` (Space variable) so operator
        or Hermes-UI edits are not overwritten on every boot. Obsolete chains
        (e.g. ``openrouter/free``) are always refreshed regardless of preserve.
        """
        config_path = HERMES_DATA / "config.yaml"
        try:
            import yaml
            data = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {}
            preserve = os.environ.get(
                "HERMES_PRESERVE_FALLBACK_PROVIDERS", "false"
            ).strip().lower() in {"1", "true", "yes", "on"}
            desired_chain = build_advisory_fallback_chain()
            auxiliary = data.setdefault("auxiliary", {})
            if not isinstance(auxiliary, dict):
                auxiliary = {}
                data["auxiliary"] = auxiliary
            existing_aux = auxiliary.get(FUTURES_ADVISORY_TASK)
            existing_fb = data.get("fallback_providers")
            if (
                preserve
                and desired_chain
                and advisory_config_is_current(existing_fb, existing_aux, desired_chain)
            ):
                labels = advisory_chain_labels(desired_chain)
                print(
                    f"[SYNC] Advisory provider config preserved "
                    f"(fallback_providers + auxiliary.{FUTURES_ADVISORY_TASK}): {labels}"
                )
                return

            if preserve and is_obsolete_advisory_chain(existing_fb):
                print(
                    "[SYNC] Advisory provider config refresh: "
                    "obsolete fallback_providers detected (preserve overridden)"
                )

            if desired_chain:
                data["fallback_providers"] = [dict(entry) for entry in desired_chain]
                auxiliary[FUTURES_ADVISORY_TASK] = futures_advisory_auxiliary_config(
                    desired_chain
                )
            else:
                data.pop("fallback_providers", None)
                auxiliary.pop(FUTURES_ADVISORY_TASK, None)

            config_path.write_text(yaml.safe_dump(data, sort_keys=False), encoding="utf-8")
            labels = advisory_chain_labels(desired_chain)
            print(f"[SYNC] fallback_providers configured: {labels or 'none (no credentials)'}")
            if desired_chain:
                print(
                    f"[SYNC] auxiliary.{FUTURES_ADVISORY_TASK} persisted "
                    f"(provider=auto, fallback_chain={len(desired_chain)} steps)"
                )
        except Exception as e:
            print(f"[SYNC] WARNING: could not write fallback_providers: {e}")

    def _park_unavailable_optional_mcp(self):
        """Keep optional local MCP servers from entering a reconnect loop.

        Unreal Engine and Linear are opt-in in a headless Space. Linear also
        requires cached OAuth credentials before it is exposed to child
        processes; otherwise one sanitized warning is emitted and the entry is
        parked for this boot. All other MCP entries remain untouched.
        """
        config_path = HERMES_DATA / "config.yaml"
        try:
            import yaml
            data = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {}
            policy = apply_optional_mcp_policy(
                data,
                linear_enabled=os.environ.get("LINEAR_MCP_ENABLED", "false").strip().lower() == "true",
                linear_tokens_available=(HERMES_DATA / "mcp-tokens" / "linear.json").is_file(),
                unreal_enabled=os.environ.get("UNREAL_ENGINE_MCP_ENABLED", "false").lower() in {"1", "true", "yes", "on"},
            )
            if policy["removed"]:
                config_path.write_text(yaml.safe_dump(data, sort_keys=False), encoding="utf-8")
            for name in policy["removed"]:
                if name.lower() == "unreal-engine":
                    print("[SYNC] Optional unreal-engine MCP parked (not enabled)")
                elif name.lower() == "linear" and not policy["linear_unconfigured"]:
                    print("[SYNC] Linear MCP disabled; OAuth discovery skipped")
            if policy["linear_unconfigured"]:
                print("[SYNC] WARNING: Linear MCP enabled but no cached OAuth credentials are available; parked for this process")
        except Exception as exc:
            print(f"[SYNC] Optional MCP parking skipped: {exc}")

    def _disable_legacy_telegram_gateway(self):
        """Make the Gateway's persisted platform map agree with webhook mode.

        Hermes loads ~/.hermes/.env after the child environment is built, so
        removing TELEGRAM_BOT_TOKEN from one env dict is insufficient. An
        explicit YAML ``enabled: false`` is the authoritative propagation
        point and leaves the dashboard webhook process independent.
        """
        enabled = os.environ.get("TELEGRAM_ENABLED", "false").strip().lower() == "true"
        mode = os.environ.get("TELEGRAM_MODE", "webhook").strip().lower()
        if enabled and mode != "webhook":
            return
        config_path = HERMES_DATA / "config.yaml"
        try:
            import yaml
            data = yaml.safe_load(config_path.read_text(encoding="utf-8")) or {}
            platforms = data.setdefault("platforms", {})
            if isinstance(platforms, dict):
                telegram = platforms.setdefault("telegram", {})
                if isinstance(telegram, dict):
                    telegram["enabled"] = False
                    config_path.write_text(yaml.safe_dump(data, sort_keys=False), encoding="utf-8")
                    if enabled and mode == "webhook":
                        print("[SYNC] Telegram runtime: webhook-only; Gateway polling adapter disabled")
                    else:
                        print("[SYNC] Telegram runtime: disabled; no adapter or reconnect task started")
        except Exception as exc:
            print(f"[SYNC] Telegram Gateway isolation skipped: {exc}")

    # ── Background sync loop ──────────────────────────────────────────

    def background_sync_loop(self, stop_event):
        print(f"[SYNC] Background sync started (interval={SYNC_INTERVAL}s)")
        while not stop_event.is_set():
            if stop_event.wait(timeout=SYNC_INTERVAL):
                break
            print(f"[SYNC] Periodic sync triggered at {datetime.now().isoformat()}")
            self.save_to_repo()

    # ── Application runner ─────────────────────────────────────────────

    def _patch_web_server_cors(self):
        """Apply bounded security/header changes to the pinned Hermes server.

        The patch allows the official Hugging Face embedding origins and local
        development only. Framing is controlled by CSP; the legacy X-Frame
        header is renamed rather than weakened to an invalid ALLOWALL value.
        Every mutation is exact-count checked against the immutable upstream.
        """
        ws_path = APP_DIR / "hermes_cli" / "web_server.py"
        if not ws_path.exists():
            raise RuntimeError(f"required upstream web server missing: {ws_path}")
        try:
            code = ws_path.read_text(encoding="utf-8")
            changed = False

            old_cors = r'allow_origin_regex=r"^https?://(localhost|127\.0\.0\.1)(:\d+)?$"'
            new_cors = (
                r'allow_origin_regex=r"^(https://huggingface\.co|'
                r'https://[A-Za-z0-9-]+\.hf\.space|'
                r'https?://(localhost|127\.0\.0\.1)(:\d+)?)$"'
            )
            code, did = _replace_exact_once(code, old_cors, new_cors, label="web CORS patch")
            changed = changed or did
            if did:
                print("[SYNC] Restricted web_server.py CORS to HF Spaces and loopback")

            for value in ("DENY", "SAMEORIGIN"):
                old = f'X-Frame-Options", "{value}"'
                new = f'X-HermesFace-Legacy-Frame-Options", "{value}"'
                code, did = _replace_exact_once(code, old, new, label=f"frame header {value}")
                changed = changed or did
                if did:
                    print("[SYNC] Delegated iframe policy to CSP frame-ancestors")

            csp_old = "frame-ancestors 'none'"
            csp_new = "frame-ancestors 'self' https://huggingface.co https://*.hf.space"
            code, did = _replace_exact_once(code, csp_old, csp_new, label="CSP frame-ancestors patch")
            changed = changed or did
            if did:
                print("[SYNC] Restricted CSP frame-ancestors to self and Hugging Face")

            health_marker = '@app.get("/health")\nasync def _hermesface_health'
            old_status_route = '@app.get("/api/status")\nasync def get_status('
            if health_marker not in code and old_status_route in code:
                new_routes = (
                    '@app.get("/health")\n'
                    'async def _hermesface_health():\n'
                    '    """HermesFace process liveness alias."""\n'
                    '    return await get_status()\n\n\n'
                    + old_status_route
                )
                code, did = _replace_exact_once(
                    code, old_status_route, new_routes, label="health route patch"
                )
                changed = changed or did

            if changed:
                ws_path.write_text(code, encoding="utf-8")
        except Exception as exc:
            print(f"[SYNC] web_server security patch failed: {type(exc).__name__}: {exc}")
            raise

    def _install_futures_overlay(self):
        """Copy the HermesFace Futures trading overlay into /opt/hermes.

        Source lives at /opt/hermesface_overlay (COPY'd from this repo's
        `hermes_overlay/` at build time, see Dockerfile). The Futures agent
        toolset registers via ``$HERMES_HOME/plugins/futures_trading/`` using
        Hermes' plugin system. Dashboard routers and the deterministic
        ``trading/`` package still live under /opt/hermes.
        """
        # Keep the deployed source outside /opt/data: persistence restore may
        # contain an older hermes_overlay snapshot and must never downgrade
        # the code shipped in the current Space image.
        overlay_src = Path("/opt/hermesface_overlay")
        if not overlay_src.exists():
            overlay_src = HERMES_DATA / "hermes_overlay"
        if not overlay_src.exists():
            raise RuntimeError("HermesFace Futures overlay is missing from the image")
        try:
            trading_src = overlay_src / "trading"
            tools_src = overlay_src / "tools"
            if not trading_src.is_dir() or not tools_src.is_dir():
                raise RuntimeError("HermesFace overlay is incomplete: trading/ and tools/ are required")
            shutil.copytree(trading_src, APP_DIR / "trading", dirs_exist_ok=True)
            for f in tools_src.glob("*.py"):
                shutil.copy2(f, APP_DIR / "tools" / f.name)
            plugins_src = overlay_src / "plugins" / "futures_trading"
            plugins_dest = HERMES_DATA / "plugins" / "futures_trading"
            if plugins_src.is_dir():
                plugins_dest.parent.mkdir(parents=True, exist_ok=True)
                shutil.copytree(plugins_src, plugins_dest, dirs_exist_ok=True)
                print("[SYNC] Installed futures_trading plugin into $HERMES_HOME/plugins/")
            skills_src = overlay_src / "skills"
            if skills_src.is_dir():
                skills_dest = HERMES_DATA / "skills"
                skills_dest.mkdir(parents=True, exist_ok=True)
                for skill_dir in skills_src.rglob("SKILL.md"):
                    rel = skill_dir.parent.relative_to(skills_src)
                    target = skills_dest / rel
                    target.mkdir(parents=True, exist_ok=True)
                    for item in skill_dir.parent.iterdir():
                        dest_item = target / item.name
                        if item.is_dir():
                            shutil.copytree(item, dest_item, dirs_exist_ok=True)
                        else:
                            shutil.copy2(item, dest_item)
                print("[SYNC] Installed Futures skills into $HERMES_HOME/skills/")
            external_ai_src = overlay_src / "external_ai"
            if external_ai_src.exists():
                shutil.copytree(external_ai_src, APP_DIR / "external_ai", dirs_exist_ok=True)
            templates_src = tools_src / "templates"
            if templates_src.exists():
                shutil.copytree(templates_src, APP_DIR / "tools" / "templates", dirs_exist_ok=True)
            tracked = {
                "router": (
                    tools_src / "futures_dashboard_api.py",
                    APP_DIR / "tools" / "futures_dashboard_api.py",
                ),
                "template": (
                    templates_src / "hermes_futures_desk_luxury.html",
                    APP_DIR / "tools" / "templates" / "hermes_futures_desk_luxury.html",
                ),
            }

            def _sha256(path):
                if not path.is_file():
                    return None
                digest = hashlib.sha256()
                with path.open("rb") as handle:
                    for chunk in iter(lambda: handle.read(1024 * 1024), b""):
                        digest.update(chunk)
                return digest.hexdigest()

            manifest = {
                "installedAt": datetime.now().astimezone().isoformat(),
                "sourceRoot": str(overlay_src),
                "destinationRoot": str(APP_DIR),
                "files": {},
            }
            for name, (source_path, destination_path) in tracked.items():
                source_hash = _sha256(source_path)
                destination_hash = _sha256(destination_path)
                manifest["files"][name] = {
                    "sourcePath": str(source_path),
                    "destinationPath": str(destination_path),
                    "sourceSha256": source_hash,
                    "destinationSha256": destination_hash,
                    "matches": bool(source_hash and source_hash == destination_hash),
                }
            manifest_path = APP_DIR / ".hermes_futures_overlay_manifest.json"
            manifest_path.write_text(json.dumps(manifest, indent=2), encoding="utf-8")
            mismatch = [name for name, item in manifest["files"].items() if not item["matches"]]
            if mismatch:
                raise RuntimeError(f"Futures overlay hash mismatch: {', '.join(mismatch)}")
            print("[SYNC] Futures overlay hashes verified")
            print("[SYNC] Futures trading overlay installed into /opt/hermes")
        except Exception as e:
            print(f"[SYNC] Futures overlay install failed: {e}")
            raise

    def _patch_web_server_futures_dashboard(self):
        """Mount the Futures dashboard router onto the existing FastAPI `app`.

        Same server, same port, same process as the rest of HermesFace --
        this is the additive "extend the dashboard" hook, not a separate
        frontend. See tools/futures_dashboard_api.py for the actual routes
        (/api/futures/status, /api/futures/positions, /futures).
        """
        ws_path = APP_DIR / "hermes_cli" / "web_server.py"
        if not ws_path.exists():
            raise RuntimeError(f"required upstream web server missing: {ws_path}")
        try:
            code = ws_path.read_text(encoding="utf-8")
            if "futures_dashboard_api" in code and "telegram_bot" in code:
                return  # already patched
            marker = 'app = FastAPI(title="Hermes Agent", version=__version__, lifespan=_lifespan)\n'
            patch = (
                marker
                + "from tools.futures_dashboard_api import router as _futures_dashboard_router\n"
                + "app.include_router(_futures_dashboard_router)\n"
                + "from tools.telegram_bot import router as _telegram_router\n"
                + "app.include_router(_telegram_router)\n"
            )
            code = _replace_required_once(code, marker, patch, label="Futures dashboard router mount")
            ws_path.write_text(code, encoding="utf-8")
            print("[SYNC] Patched web_server.py: mounted Futures and Telegram webhook routers")
        except Exception as e:
            print(f"[SYNC] web_server.py Futures dashboard patch failed: {e}")
            raise

    def _patch_auxiliary_client_fallback(self):
        """Fix configured fallback-chain selection and enforce OpenRouter :free guard."""
        try:
            from patch_auxiliary_client import apply_auxiliary_client_fallback_patch
        except ImportError:
            from scripts.patch_auxiliary_client import apply_auxiliary_client_fallback_patch  # type: ignore

        aux_path = APP_DIR / "agent" / "auxiliary_client.py"
        try:
            changes = apply_auxiliary_client_fallback_patch(aux_path)
            for item in changes:
                print(f"[SYNC] auxiliary_client patch: {item}")
        except Exception as exc:
            print(f"[SYNC] auxiliary_client fallback patch failed: {exc}")
            raise

    def _patch_telegram_webhook_public_path(self):
        """Let Telegram reach its own HMAC-protected webhook before cookie auth."""
        path = APP_DIR / "hermes_cli" / "dashboard_auth" / "public_paths.py"
        if not path.exists():
            raise RuntimeError(f"required upstream dashboard public-path registry missing: {path}")
        try:
            code = path.read_text(encoding="utf-8")
            marker = '    "/api/cron/fire",\n'
            additions = []
            if '"/api/telegram/webhook"' not in code:
                additions.append('    "/api/telegram/webhook",  # HMAC-protected Telegram ingress\n')
            if '"/api/telegram/bootstrap/status"' not in code:
                additions.append('    "/api/telegram/bootstrap/status",  # secret-protected activation status\n')
            if not additions:
                return
            code = _replace_required_once(
                code, marker, marker + "".join(additions), label="Telegram public ingress paths"
            )
            path.write_text(code, encoding="utf-8")
            print("[SYNC] Added HMAC-protected Telegram webhook to public ingress paths")
        except Exception as exc:
            print(f"[SYNC] Telegram webhook auth-path patch failed: {exc}")
            raise

    def _start_process(self, cmd, label, env, log_path):
        """Helper to start a subprocess with output logging."""
        log_fh = open(log_path, "a")
        try:
            process = subprocess.Popen(
                cmd,
                cwd=str(APP_DIR),
                stdout=subprocess.PIPE,
                stderr=subprocess.STDOUT,
                text=True,
                bufsize=1,
                env=env,
            )

            def copy_output():
                try:
                    for line in process.stdout:
                        log_fh.write(line)
                        log_fh.flush()
                        stripped = line.strip()
                        if not stripped:
                            continue
                        if any(skip in stripped for skip in [
                            'Downloading', 'Fetching', '%|', '━', '───',
                            'Already cached', 'Using cache', 'tokenizer',
                            '.safetensors', 'model-', 'shard',
                        ]):
                            continue
                        print(line, end='')
                except Exception as e:
                    print(f"[SYNC] {label} output error: {e}")
                finally:
                    log_fh.close()

            threading.Thread(target=copy_output, daemon=True).start()
            print(f"[SYNC] {label} started (PID {process.pid})")
            return process
        except Exception as e:
            log_fh.close()
            print(f"[SYNC] ERROR starting {label}: {e}")
            traceback.print_exc()
            return None

    def run_hermes(self):
        """Start Hermes: web dashboard on port 7860, gateway in background if messaging tokens configured."""
        log_dir = HERMES_DATA / "logs"
        log_dir.mkdir(parents=True, exist_ok=True)

        if not APP_DIR.exists():
            print(f"[SYNC] ERROR: App directory does not exist: {APP_DIR}")
            return None

        hermes_bin = shutil.which("hermes") or str(APP_DIR / ".venv" / "bin" / "hermes")
        if not Path(hermes_bin).exists():
            print("[SYNC] ERROR: hermes CLI not found")
            return None

        self._configure_admin_auth()
        self._configure_audit_signing_secret()
        env = os.environ.copy()
        env["HERMES_HOME"] = str(HERMES_DATA)
        env["GATEWAY_ALLOW_ALL_USERS"] = "true"
        # Prevent gateway from grabbing port 7860
        env.pop("API_SERVER_ENABLED", None)
        env.pop("API_SERVER_PORT", None)
        # Telegram is an isolated optional platform. It is opt-in for the
        # Space so regional egress failures cannot create reconnect storms.
        if env.get("TELEGRAM_ENABLED", "false").lower() not in {"1", "true", "yes", "on"}:
            env.pop("TELEGRAM_BOT_TOKEN", None)
            print("[SYNC] Telegram platform disabled (set TELEGRAM_ENABLED=true to enable)")
        self._park_unavailable_optional_mcp()
        self._disable_legacy_telegram_gateway()

        # ── 1. Install the HermesFace Futures overlay + patch dashboard ──
        self._install_futures_overlay()
        self._patch_web_server_cors()
        self._patch_telegram_webhook_public_path()
        self._patch_web_server_futures_dashboard()
        self._patch_auxiliary_client_fallback()

        # ── 2. Start web dashboard on port 7860 (HF Spaces frontend) ─
        # Public binds now always require an auth provider. Basic auth is wired
        # above, so the deprecated/no-op --insecure flag must not be passed.
        # The web bundle was built in the Docker image; skip an unnecessary
        # runtime npm rebuild to improve cold-start latency.
        dashboard_cmd = [
            hermes_bin, "dashboard", "--host", "0.0.0.0", "--port", "7860",
            "--no-open", "--skip-build",
        ]
        print("[SYNC] Starting web dashboard on port 7860...")
        dashboard_proc = self._start_process(
            dashboard_cmd, "Dashboard", env, log_dir / "dashboard.log"
        )

        # ── 3. Keep the public Space dashboard single-process by default ──
        # The gateway is an optional messaging/cron worker. Starting it on
        # every boot added an unnecessary second long-running process and made
        # the desktop flap when the optional platform adapters/reconnect loops
        # were unavailable. It is now opt-in for operators who explicitly need
        # gateway features; the Futures dashboard remains the only default
        # listener on a Hugging Face Space.
        gateway_enabled = os.environ.get("HERMES_GATEWAY_ENABLED", "false").strip().lower() in {
            "1", "true", "yes", "on"
        }
        if not gateway_enabled:
            self.gateway_proc = None
            print("[SYNC] Gateway disabled (set HERMES_GATEWAY_ENABLED=true to enable)")
        else:
            time.sleep(2)  # Let dashboard bind 7860 first
            gateway_env = env.copy()
            gateway_env["GATEWAY_ALLOW_ALL_USERS"] = "false"
            # Webhook mode owns Telegram updates; never start Hermes' polling
            # adapter alongside it. The dashboard process still retains the token
            # for optional direct replies/relay delivery.
            if gateway_env.get("TELEGRAM_MODE", "webhook").lower() == "webhook":
                gateway_env["TELEGRAM_ENABLED"] = "false"
                gateway_env.pop("TELEGRAM_BOT_TOKEN", None)
            gateway_cmd = [hermes_bin, "gateway"]
            print("[SYNC] Starting gateway (HERMES_GATEWAY_ENABLED=true)...")
            self.gateway_proc = self._start_process(
                gateway_cmd, "Gateway", gateway_env, log_dir / "gateway.log"
            )

        return dashboard_proc


# ── Main ────────────────────────────────────────────────────────────────────

def main():
    try:
        t_main_start = time.time()

        t0 = time.time()
        sync = HermesFullSync()
        print(f"[TIMER] sync_hf init: {time.time() - t0:.1f}s")

        # 1. Restore
        t0 = time.time()
        sync.load_from_repo()
        print(f"[TIMER] load_from_repo (restore): {time.time() - t0:.1f}s")

        # 2. Background sync
        stop_event = threading.Event()
        t = threading.Thread(target=sync.background_sync_loop, args=(stop_event,), daemon=True)
        t.start()

        # 3. Start application (Hermes API server will bind port 7860)
        t0 = time.time()
        process = sync.run_hermes()
        print(f"[TIMER] run_hermes launch: {time.time() - t0:.1f}s")
        print(f"[TIMER] Total startup (init → app launched): {time.time() - t_main_start:.1f}s")

        # Signal handler
        def handle_signal(sig, frame):
            print(f"\n[SYNC] Signal {sig} received. Shutting down...")
            stop_event.set()
            t.join(timeout=10)
            # Stop gateway
            if hasattr(sync, 'gateway_proc') and sync.gateway_proc:
                sync.gateway_proc.terminate()
                try:
                    sync.gateway_proc.wait(timeout=5)
                except subprocess.TimeoutExpired:
                    sync.gateway_proc.kill()
            # Stop dashboard
            if process:
                process.terminate()
                try:
                    process.wait(timeout=5)
                except subprocess.TimeoutExpired:
                    process.kill()
            print("[SYNC] Final sync...")
            sync.save_to_repo()
            sys.exit(0)

        signal.signal(signal.SIGINT, handle_signal)
        signal.signal(signal.SIGTERM, handle_signal)

        # Wait
        if process is None:
            print("[SYNC] ERROR: Failed to start Hermes process. Exiting.")
            stop_event.set()
            t.join(timeout=5)
            sys.exit(1)

        exit_code = process.wait()
        print(f"[SYNC] Hermes exited with code {exit_code}")
        stop_event.set()
        t.join(timeout=10)
        print("[SYNC] Final sync...")
        sync.save_to_repo()
        sys.exit(exit_code)

    except Exception as e:
        print(f"[SYNC] FATAL ERROR in main: {e}")
        traceback.print_exc()
        sys.exit(1)


if __name__ == "__main__":
    main()