#!/usr/bin/env python3 """Control server for HF Space backup management. Provides a REST API for controlling OpenViking, gateway status, and triggering backups. Runs on port 4000 with only Python 3 built-ins. """ import http.server import json import mimetypes import os import signal import subprocess import threading import time import urllib.parse from datetime import datetime # ============================================================ # Constants # ============================================================ CONTROL_DIR = os.path.dirname(os.path.abspath(__file__)) BACKUP_LOG = "/tmp/backup-control.log" BACKUP_MANAGER = os.path.join(os.path.dirname(os.path.abspath(__file__)), "backup-manager.py") OPENVIKING_PID_FILE = "/tmp/openviking.pid" OPENVIKING_CONFIG = os.path.join( os.environ.get("HOME", "/root"), ".openviking", "ov.conf" ) GATEWAY_PORT = os.environ.get("OPENCLAW_GATEWAY_PORT", os.environ.get("PORT", "7860")) STATE_DIR = os.environ.get("OPENCLAW_STATE_DIR", "/root/.openclaw") SERVER_PORT = 4000 OPENVIKING_PORT = 1933 # HTTP limits MAX_BODY_SIZE = 10 * 1024 * 1024 # 10 MB DEFAULT_MAX_CONCURRENT = 100 # Timeouts BACKUP_CMD_TIMEOUT = 600 # 10 minutes (was 1 hour) CURL_TIMEOUT = 5 # seconds STOP_WAIT_ITERATIONS = 20 STOP_WAIT_INTERVAL = 0.5 # seconds LOG_MAX_LINES = 200 LOG_MAX_BYTES = 20 * 1024 # 20 KB # ============================================================ # Synchronization # ============================================================ _backup_lock = threading.Lock() # ============================================================ # Logging # ============================================================ def write_log(message: str) -> None: """Append a timestamped entry to the backup control log.""" timestamp = datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") line = f"{timestamp} {message}\n" try: with open(BACKUP_LOG, "a", encoding="utf-8") as f: f.write(line) except OSError as e: print(f"Log write error: {e}") def write_separator() -> None: """Write a separator line to the backup control log.""" write_log("=" * 40) def _read_log_tail(max_lines: int = LOG_MAX_LINES, max_bytes: int = LOG_MAX_BYTES) -> list[str]: """读取日志文件尾部最多 max_lines 行(最多读取 max_bytes 字节).""" try: log_size = os.path.getsize(BACKUP_LOG) read_size = min(log_size, max_bytes) with open(BACKUP_LOG, "rb") as f: if read_size < log_size: f.seek(log_size - read_size) f.readline() # skip partial first line raw = f.read().decode("utf-8", errors="replace") all_lines = raw.splitlines() return [l.rstrip("\r\n") for l in all_lines[-max_lines:]] except (FileNotFoundError, OSError): return [] # ============================================================ # Process helpers # ============================================================ def pid_is_alive(pid_file: str) -> bool: """Return True if the process identified by *pid_file* is running. Uses ``kill -0`` which never sends a signal but checks existence. """ try: with open(pid_file, "r") as f: pid = int(f.read().strip()) os.kill(pid, 0) return True except (FileNotFoundError, ValueError, ProcessLookupError): return False except OSError: return False def port_is_alive(url: str) -> bool: """Return True if the given URL responds successfully via curl.""" try: result = subprocess.run( ["curl", "-sf", url], capture_output=True, timeout=CURL_TIMEOUT, ) return result.returncode == 0 except (subprocess.TimeoutExpired, FileNotFoundError): return False # ============================================================ # OpenViking lifecycle # ============================================================ def stop_openviking() -> bool: """Stop the OpenViking server. 1. Read PID from ``/tmp/openviking.pid``. 2. Send SIGTERM and wait up to 10 seconds. 3. If still alive, send SIGKILL. """ if not os.path.exists(OPENVIKING_PID_FILE): write_log("OpenViking PID file not found, assuming already stopped") return True try: with open(OPENVIKING_PID_FILE, "r") as f: pid = int(f.read().strip()) except (ValueError, OSError) as e: write_log(f"Cannot read OpenViking PID: {e}") return True write_log(f"Stopping OpenViking (PID: {pid})") # SIGTERM try: os.kill(pid, signal.SIGTERM) except ProcessLookupError: write_log("OpenViking process not found") return True except OSError as e: write_log(f"Error sending SIGTERM: {e}") # Wait for graceful shutdown for _ in range(STOP_WAIT_ITERATIONS): try: os.kill(pid, 0) time.sleep(STOP_WAIT_INTERVAL) except ProcessLookupError: write_log("OpenViking stopped gracefully") return True except OSError: time.sleep(0.5) # SIGKILL write_log("OpenViking did not stop gracefully, sending SIGKILL") try: os.kill(pid, signal.SIGKILL) time.sleep(STOP_WAIT_INTERVAL) except OSError as e: write_log(f"Error sending SIGKILL: {e}") return False return True def start_openviking() -> bool: """Start the OpenViking server and write its PID file. Requires ``openviking-server`` on PATH and ``~/.openviking/ov.conf``. """ if not os.path.exists(OPENVIKING_CONFIG): write_log(f"OpenViking config not found: {OPENVIKING_CONFIG}") return False env = os.environ.copy() try: with open("/tmp/openviking-server.log", "w") as log_f: proc = subprocess.Popen( ["nohup", "openviking-server", "--config", OPENVIKING_CONFIG], stdout=log_f, stderr=subprocess.STDOUT, env=env, preexec_fn=os.setpgrp, ) except FileNotFoundError: write_log("openviking-server binary not found in PATH") return False except OSError as e: write_log(f"Failed to start OpenViking: {e}") return False with open(OPENVIKING_PID_FILE, "w") as f: f.write(str(proc.pid)) write_log(f"OpenViking started (PID: {proc.pid})") return True # ============================================================ # Gateway lifecycle # ============================================================ def restart_gateway() -> bool: """重启 OpenClaw Gateway(写标记文件 → 杀进程 → 等待新实例就绪)""" import signal pid_file = "/tmp/openclaw-gateway.pid" restart_marker = "/tmp/openclaw-gateway-restart" gateway_port = os.environ.get("OPENCLAW_INTERNAL_PORT") if not gateway_port: base_port = int(os.environ.get("PORT", "7860")) gateway_port = str(base_port + 1) if not os.path.exists(pid_file): write_log("Gateway PID file not found") return False try: with open(pid_file) as f: old_pid = int(f.read().strip()) # 写重启标记 → 启动脚本检测到后会自动重启新实例 with open(restart_marker, "w") as f: f.write("1") # 杀旧进程 os.kill(old_pid, signal.SIGTERM) write_log(f"Gateway SIGTERM sent to PID {old_pid}") # 等待旧进程退出 for _ in range(20): try: os.kill(old_pid, 0) time.sleep(0.5) except ProcessLookupError: break # 等待新实例就绪(最多60秒) for i in range(60): time.sleep(1) # 新进程的 PID 可能变了,直接检查端口 try: result = subprocess.run( ["curl", "-sf", f"http://127.0.0.1:{gateway_port}/health"], capture_output=True, timeout=5, ) if result.returncode == 0: # 读新 PID if os.path.exists(pid_file): with open(pid_file) as f: new_pid = f.read().strip() write_log(f"Gateway restarted (PID: {new_pid}) on :{gateway_port}") else: write_log("Gateway restarted on :{gateway_port}") return True except Exception: pass write_log("Gateway restart timed out (60s)") return False except (FileNotFoundError, ValueError, ProcessLookupError, OSError) as e: write_log(f"Gateway restart failed: {e}") return False def restart_space(factory=False) -> bool: """通过 HF API 重启/重构整个 Space。factory=True 时上传空标记触发完整重构。""" write_log("Restarting Space via HF API...") try: from huggingface_hub import HfApi token = os.environ.get("HF_TOKEN", "") api = HfApi() if factory: # 上传空标记文件触发新 commit → HF 自动触发完整重建 import datetime marker = f"rebuild-{datetime.datetime.now().strftime('%Y%m%d-%H%M%S')}" api.upload_file( path_or_fileobj=marker.encode(), path_in_repo=".rebuild-marker", repo_id="sharween/xiaoxiaxia", repo_type="space", token=token, ) write_log("Space rebuild triggered via commit") else: api.restart_space("sharween/xiaoxiaxia", token=token) write_log("Space restart initiated") return True except Exception as e: write_log(f"Space restart failed: {e}") return False # ============================================================ # Backup operations # ============================================================ _BACKUP_LABELS = { "quick": "增量备份", "full": "全量备份", "webdav_quick": "WEBDAV 增量备份", "webdav_full": "WEBDAV 全量备份", "dataset_quick": "DATASET 小备份", "dataset_full": "DATASET 大备份", } def _run_backup_cmd(cmd_args: list[str], timeout: int = BACKUP_CMD_TIMEOUT) -> bool: """Execute backup-manager.py and log output.""" cmd = ["python3", BACKUP_MANAGER] + cmd_args write_log(f"Running: {' '.join(cmd)}") try: result = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout) if result.stdout: for line in result.stdout.strip().split("\n"): if line.strip(): write_log(line) if result.stderr: for line in result.stderr.strip().split("\n"): if line.strip(): write_log(f"[stderr] {line}") if result.returncode == 0: write_log("Backup command completed successfully") else: write_log(f"Backup command failed with code {result.returncode}") return result.returncode == 0 except subprocess.TimeoutExpired: write_log("Backup command timed out") return False except FileNotFoundError: write_log(f"backup-manager.py not found at {BACKUP_MANAGER}") return False except OSError as e: write_log(f"Backup command error: {e}") return False def _background_task(name: str, cmd_args: list[str]) -> None: """Run a backup command in a background daemon thread. 所有备份都停 OV → 执行 → 启 OV(保证 vectordb 一致性). """ def _task() -> None: if not _backup_lock.acquire(blocking=False): write_log(f"{name} already in progress, request ignored") return try: write_separator() write_log(f"Starting {name}...") stop_openviking() _run_backup_cmd(cmd_args) start_openviking() write_log(f"{name} completed") write_separator() except Exception as e: write_log(f"{name} failed: {e}") write_separator() finally: _backup_lock.release() thread = threading.Thread(target=_task, daemon=True) thread.start() def background_webdav_backup() -> None: _background_task("WEBDAV 增量备份", ["incremental"]) def background_webdav_full_backup() -> None: _background_task("WEBDAV 全量备份", ["full"]) def background_dataset_backup() -> None: _background_task("DATASET 小备份", ["small"]) def background_dataset_full_backup() -> None: _background_task("DATASET 大备份", ["large"]) # ============================================================ # HTTP handler # ============================================================ class BackupHandler(http.server.BaseHTTPRequestHandler): """HTTP request handler for the backup control server.""" # Suppress default per-request stderr logging def log_message(self, fmt: str, *args) -> None: pass # ---- helpers ------------------------------------------------------- def _send_json(self, data, status: int = 200) -> None: body = json.dumps(data, ensure_ascii=False).encode("utf-8") self.send_response(status) self.send_header("Content-Type", "application/json; charset=utf-8") self.send_header("Access-Control-Allow-Origin", "*") self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS") self.send_header("Access-Control-Allow-Headers", "Content-Type") self.send_header("Content-Length", str(len(body))) self.end_headers() self.wfile.write(body) def _serve_file(self, file_path: str) -> None: content_type, _ = mimetypes.guess_type(file_path) if content_type is None: content_type = "application/octet-stream" try: with open(file_path, "rb") as f: content = f.read() except OSError as e: self._send_json({"error": f"Failed to read file: {e}"}, 500) return self.send_response(200) self.send_header("Content-Type", content_type) self.send_header("Content-Length", str(len(content))) self.send_header("Access-Control-Allow-Origin", "*") self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS") self.send_header("Access-Control-Allow-Headers", "Content-Type") self.end_headers() self.wfile.write(content) # ---- CORS preflight ------------------------------------------------ def do_OPTIONS(self) -> None: self.send_response(204) self.send_header("Access-Control-Allow-Origin", "*") self.send_header("Access-Control-Allow-Methods", "GET, POST, OPTIONS") self.send_header("Access-Control-Allow-Headers", "Content-Type") self.end_headers() # ---- GET routes ---------------------------------------------------- def do_GET(self) -> None: parsed = urllib.parse.urlparse(self.path) path = parsed.path if path == "/api/status": self._handle_status() elif path == "/api/config": self._handle_config() elif path == "/api/models": self._handle_models() elif path == "/api/log": self._handle_log() elif path == "/api/backup/check": self._handle_backup_check() elif path == "/api/backup/list": self._handle_backup_list() else: self._handle_static(path) def _handle_status(self) -> None: ov_running = pid_is_alive(OPENVIKING_PID_FILE) or port_is_alive(f"http://127.0.0.1:{OPENVIKING_PORT}/health") gw_running = port_is_alive(f"http://127.0.0.1:{GATEWAY_PORT}/health") # Check whether a backup thread is currently running bk_running = not _backup_lock.acquire(blocking=False) if not bk_running: _backup_lock.release() # Determine the last successful backup time from the log (read tail only) last_backup = None tail_lines = _read_log_tail(max_lines=200, max_bytes=50 * 1024) for line in reversed(tail_lines): if line.startswith("[") and "] " in line: ts_end = line.index("]") msg = line[ts_end + 2:] # strip off "[ts] " if "Backup command completed successfully" in msg or \ "Full backup completed" in msg: last_backup = line[1:ts_end] break self._send_json({ "openviking": {"status": ("running" if ov_running else "stopped")}, "openclaw": {"status": ("running" if gw_running else "stopped")}, "last_backup": last_backup, "backup_running": bk_running, }) def _handle_config(self) -> None: space_id = os.environ.get("SPACE_ID", "sharween/xiaoxiaxia") self._send_json({"space_id": space_id}) def _handle_models(self) -> None: models = [] for i in range(1, 7): model_id = os.environ.get(f"MODEL{i}_ID", "").strip() base_url = os.environ.get(f"MODEL{i}_BASEURL", "").strip().rstrip("/") api_key = os.environ.get(f"MODEL{i}_APIKEY", "").strip() if model_id and base_url and api_key: models.append({ "index": i, "id": model_id, "name": os.environ.get(f"MODEL{i}_NAME", model_id), "base_url": base_url, "has_key": bool(api_key), }) # Also get current VLM config from ov.conf current_vlm = None try: with open(os.path.expanduser("~/.openviking/ov.conf")) as f: cfg = json.load(f) current_vlm = cfg.get("vlm", {}) except Exception: write_log("Failed to read ov.conf for VLM config") self._send_json({"models": models, "current_vlm": current_vlm}) def _handle_log(self) -> None: lines = _read_log_tail(max_lines=200, max_bytes=20 * 1024) self._send_json({"lines": lines}) def _handle_backup_check(self) -> None: """查询云端备份状态(替代原来的本地标记文件)""" try: proc = subprocess.run( ["python3", BACKUP_MANAGER, "check"], capture_output=True, text=True, timeout=30, ) if proc.returncode == 0 and proc.stdout.strip(): parsed = json.loads(proc.stdout.strip()) self._send_json(parsed) else: self._send_json({"backup_today": False, "last_backup": None}) except Exception as e: write_log(f"Backup check error: {e}") self._send_json({"backup_today": False, "last_backup": None}) def _handle_backup_list(self) -> None: """返回 WebDAV 和 HF Dataset 中可用的备份文件列表.""" result = {"webdav": [], "dataset": []} try: proc = subprocess.run( ["python3", BACKUP_MANAGER, "list"], capture_output=True, text=True, timeout=30, ) if proc.returncode == 0 and proc.stdout.strip(): parsed = json.loads(proc.stdout.strip()) result["webdav"] = parsed.get("webdav", []) result["dataset"] = parsed.get("dataset", []) except Exception as e: write_log(f"Backup list error: {e}") self._send_json(result) def _handle_restore(self) -> None: """接收备份文件参数,在后台线程中执行恢复.""" try: content_length = int(self.headers.get("Content-Length", 0)) if content_length > MAX_BODY_SIZE: # 10MB limit self._send_json({"status": "error", "message": "Request body too large"}, 413) return body = json.loads(self.rfile.read(content_length)) if content_length else {} except (json.JSONDecodeError, ValueError, OSError) as e: self._send_json({"status": "error", "message": f"Invalid request body: {e}"}, 400) return source = body.get("source", "auto") filename = body.get("filename", "") if not filename: self._send_json({"status": "error", "message": "filename is required"}, 400) return def _task() -> None: if not _backup_lock.acquire(blocking=False): write_log("Restore already in progress, request ignored") return try: write_separator() write_log(f"Starting restore from {source}: {filename}...") cmd = ["python3", BACKUP_MANAGER, "restore", "--source", source, "--filename", filename] write_log(f"Running: {' '.join(cmd)}") result = subprocess.run(cmd, capture_output=True, text=True, timeout=3600) if result.stdout: for line in result.stdout.strip().split("\n"): if line.strip(): write_log(line) if result.stderr: for line in result.stderr.strip().split("\n"): if line.strip(): write_log(f"[stderr] {line}") if result.returncode == 0: write_log("Restore completed successfully") else: write_log(f"Restore failed with code {result.returncode}") write_separator() finally: _backup_lock.release() thread = threading.Thread(target=_task, daemon=True) thread.start() self._send_json({"status": "started"}) def _handle_static(self, path: str) -> None: # Default to control.html for the root path if path == "/" or not path: path = "/control.html" # Normalize and prevent directory traversal relative = os.path.normpath(path.lstrip("/")) file_path = os.path.join(CONTROL_DIR, relative) real_path = os.path.realpath(file_path) real_control = os.path.realpath(CONTROL_DIR) if not real_path.startswith(real_control + os.sep) and \ real_path != real_control: self._send_json({"error": "Forbidden"}, 403) return if os.path.isfile(file_path): self._serve_file(file_path) else: self._send_json({"error": "Not found"}, 404) # ---- POST routes --------------------------------------------------- def do_POST(self) -> None: parsed = urllib.parse.urlparse(self.path) path = parsed.path if path == "/api/openviking/stop": self._handle_stop_openviking() elif path == "/api/openviking/start": self._handle_start_openviking() elif path == "/api/backup": self._handle_backup() elif path == "/api/backup/full": self._handle_full_backup() elif path == "/api/backup/webdav/quick": self._handle_webdav_backup() elif path == "/api/backup/webdav/full": self._handle_webdav_full_backup() elif path == "/api/backup/dataset/quick": self._handle_dataset_backup() elif path == "/api/backup/dataset/full": self._handle_dataset_full_backup() elif path == "/api/openviking/vlm": self._handle_set_vlm() elif path == "/api/restore": self._handle_restore() elif path == "/api/gateway/restart": self._handle_gateway_restart() elif path == "/api/system/restart": self._handle_system_restart() elif path == "/api/system/rebuild": self._handle_system_rebuild() else: self._send_json({"error": "Not found"}, 404) def _handle_stop_openviking(self) -> None: stop_openviking() self._send_json({"status": "ok", "message": "OpenViking stopped"}) def _handle_start_openviking(self) -> None: success = start_openviking() if success: self._send_json({"status": "ok", "message": "OpenViking started"}) else: self._send_json( {"status": "error", "message": "Failed to start OpenViking"}, 500, ) def _handle_backup(self) -> None: background_webdav_backup() self._send_json({"status": "started"}) def _handle_full_backup(self) -> None: background_webdav_full_backup() self._send_json({"status": "started"}) def _handle_webdav_backup(self) -> None: background_webdav_backup() self._send_json({"status": "started"}) def _handle_webdav_full_backup(self) -> None: background_webdav_full_backup() self._send_json({"status": "started"}) def _handle_dataset_backup(self) -> None: background_dataset_backup() self._send_json({"status": "started"}) def _handle_dataset_full_backup(self) -> None: background_dataset_full_backup() self._send_json({"status": "started"}) def _handle_set_vlm(self) -> None: try: content_length = int(self.headers.get("Content-Length", 0)) if content_length > MAX_BODY_SIZE: self._send_json({"status": "error", "message": "Request body too large"}, 413) return body = json.loads(self.rfile.read(content_length)) if content_length else {} except (json.JSONDecodeError, ValueError, OSError) as e: self._send_json({"status": "error", "message": f"Invalid request body: {e}"}, 400) return model_id = body.get("model", "").strip() base_url = body.get("base_url", "").strip().rstrip("/") api_key = body.get("api_key", "").strip() provider = body.get("provider", "openai") max_concurrent = body.get("max_concurrent", DEFAULT_MAX_CONCURRENT) # If model_index is provided, read config from env vars if "model_index" in body: idx = body["model_index"] model_id = os.environ.get(f"MODEL{idx}_ID", "").strip() base_url = os.environ.get(f"MODEL{idx}_BASEURL", "").strip().rstrip("/") api_key = os.environ.get(f"MODEL{idx}_APIKEY", "").strip() # Detect provider from base_url bl = base_url.lower() if "volces" in bl or "volcengine" in bl: provider = "volcengine" elif "anthropic" in bl or "claude" in model_id.lower(): provider = "litellm" else: provider = "openai" if not model_id or not base_url or not api_key: self._send_json({"status": "error", "message": "model, base_url and api_key are required"}, 400) return self._set_vlm_and_restart(model_id, base_url, api_key, provider, max_concurrent) def _set_vlm_and_restart(self, model_id, base_url, api_key, provider, max_concurrent): ov_conf_path = os.path.expanduser("~/.openviking/ov.conf") try: with open(ov_conf_path) as f: cfg = json.load(f) except Exception: cfg = { "server": {"host": "0.0.0.0", "port": 1933, "cors_origins": ["*"]}, "storage": {"workspace": "/root/.openclaw/openviking_data"}, "log": {"level": "INFO", "output": "stdout"}, } cfg["vlm"] = { "api_base": base_url, "api_key": api_key, "provider": provider, "model": model_id, "max_concurrent": max_concurrent, } os.makedirs(os.path.dirname(ov_conf_path), exist_ok=True) with open(ov_conf_path, "w") as f: json.dump(cfg, f, indent=2) stop_openviking() time.sleep(2) start_openviking() self._send_json({"status": "ok", "message": f"VLM model set to {model_id}, OpenViking restarted"}) def _handle_gateway_restart(self) -> None: success = restart_gateway() if success: self._send_json({"status": "ok", "message": "Gateway 重启成功"}) else: self._send_json({"status": "error", "message": "Gateway 重启失败,请查看日志"}, 500) def _handle_system_restart(self) -> None: self._send_json({"status": "ok", "message": "系统重启中..."}) # 等响应发送完再重启,否则容器杀死后响应不完整 threading.Timer(2.0, restart_space).start() def _handle_system_rebuild(self) -> None: self._send_json({"status": "ok", "message": "系统重构中..."}) threading.Timer(2.0, restart_space, kwargs={"factory": True}).start() # ============================================================ # Entry point # ============================================================ def main() -> None: # Ensure the log file exists try: with open(BACKUP_LOG, "a", encoding="utf-8"): pass except OSError as e: print(f"Warning: cannot create log file {BACKUP_LOG}: {e}") server = http.server.ThreadingHTTPServer( ("0.0.0.0", SERVER_PORT), BackupHandler ) write_log(f"Control server starting on port {SERVER_PORT}") print(f"Control server listening on http://0.0.0.0:{SERVER_PORT}") try: server.serve_forever() except KeyboardInterrupt: write_log("Control server shutting down") server.shutdown() if __name__ == "__main__": main()