Deploy builder with retry logic and auto-restart

app.py

@@ -168,6 +168,12 @@ class Config:
     log_format: LogFormat = field(default_factory=lambda: LogFormat(os.getenv("LOG_FORMAT", "text").lower()) if os.getenv("LOG_FORMAT", "text").lower() in ("text", "json") else LogFormat.TEXT)
     max_history: int = field(default_factory=lambda: int(os.getenv("MAX_HISTORY", "50")))
 
+    # Auto-deploy: Restart HF Space after successful build
+    # Set to HF Space ID (e.g., "username/space-name") to auto-restart on success
+    auto_restart_space: str = field(default_factory=lambda: os.getenv("AUTO_RESTART_SPACE", ""))
+    # HF Token for restarting spaces (uses HF_TOKEN env var)
+    hf_token: str = field(default_factory=lambda: os.getenv("HF_TOKEN", ""))
+
     build_patterns: list[str] = field(default_factory=lambda: [
         "Dockerfile", "Dockerfile.*", "docker/*", "docker/**/*",
         "src/**/*.py", "pyproject.toml", "uv.lock", "requirements*.txt", ".dockerignore",
@@ -400,6 +406,10 @@ class Notifier:
 
     def notify(self, build: Build) -> None:
         """Send notifications for a completed build."""
+        # Auto-restart HF Space on success
+        if build.status == BuildStatus.SUCCESS and self.config.auto_restart_space:
+            self._restart_hf_space(build)
+
         if not self.should_notify(build.status):
             return
 
@@ -415,6 +425,28 @@ class Notifier:
         if self.config.discord_webhook_url:
             self._send_discord(build)
 
+    def _restart_hf_space(self, build: Build) -> None:
+        """Restart HuggingFace Space after successful build."""
+        space_id = self.config.auto_restart_space
+        token = self.config.hf_token
+
+        if not token:
+            self.state.log(f"Cannot restart {space_id}: HF_TOKEN not set", level="warn")
+            return
+
+        try:
+            # Use HF Hub API to restart space
+            url = f"https://huggingface.co/api/spaces/{space_id}/restart"
+            headers = {"Authorization": f"Bearer {token}"}
+
+            resp = http_requests.post(url, headers=headers, json={"factory_reboot": True}, timeout=30)
+            if resp.ok:
+                self.state.log(f"✓ Restarted HF Space: {space_id}")
+            else:
+                self.state.log(f"Failed to restart {space_id}: {resp.status_code}", level="warn")
+        except Exception as e:
+            self.state.log(f"Failed to restart {space_id}: {e}", level="warn")
+
     def _send_webhook(self, build: Build) -> None:
         url = build.config.callback_url or self.config.notification_url
         if not url:
@@ -582,32 +614,90 @@ class KanikoBuilder:
         valid_prefixes = ("ghp_", "gho_", "ghu_", "ghs_", "ghr_", "github_pat_")
         return token.startswith(valid_prefixes)
 
-    def clone_repo(self, build_config: BuildConfig) -> Path:
-        target_dir = Path(tempfile.mkdtemp())
+    def clone_repo(self, build_config: BuildConfig, max_retries: int = 3) -> Path:
+        """Clone repository with retry logic and cleanup.
+
+        Retries help handle transient network issues and stale git state.
+        """
+        import git
+
         # Prefer explicit token from request, fall back to config
         token = build_config.github_token or self.config.github_token
-        repo_url = build_config.repo_url
-        use_auth = False
-
-        # Only use token if it looks valid
-        if token and self._is_valid_github_token(token) and "github.com" in repo_url:
-            repo_url = repo_url.replace("https://github.com", f"https://{token}@github.com")
-            use_auth = True
-            self.state.log(f"Cloning {build_config.repo_url} ({build_config.branch}) [authenticated]")
-        else:
-            if token and not self._is_valid_github_token(token):
-                self.state.log(f"Skipping invalid token format, trying public clone")
-            self.state.log(f"Cloning {build_config.repo_url} ({build_config.branch})")
+        base_repo_url = build_config.repo_url
 
-        try:
-            import git
-            git.Repo.clone_from(repo_url, target_dir, branch=build_config.branch, depth=1, single_branch=True)
-            self.state.log(f"Cloned to {target_dir}")
-            return target_dir
-        except Exception as e:
-            error_msg = mask_token(str(e), token) if token else str(e)
-            self.state.log(f"Clone failed: {error_msg}", level="error")
-            raise RuntimeError(f"Clone failed: {error_msg}")
+        for attempt in range(1, max_retries + 1):
+            # Fresh temp directory each attempt
+            target_dir = Path(tempfile.mkdtemp(prefix=f"build_{build_config.branch[:8]}_"))
+
+            try:
+                # Clean up any stale git locks in system temp
+                self._cleanup_stale_git_locks()
+
+                repo_url = base_repo_url
+                use_auth = False
+
+                # Only use token if it looks valid
+                if token and self._is_valid_github_token(token) and "github.com" in repo_url:
+                    repo_url = repo_url.replace("https://github.com", f"https://{token}@github.com")
+                    use_auth = True
+                    self.state.log(f"Cloning {base_repo_url} ({build_config.branch}) [authenticated] (attempt {attempt}/{max_retries})")
+                else:
+                    if token and not self._is_valid_github_token(token):
+                        self.state.log(f"Skipping invalid token format, trying public clone")
+                    self.state.log(f"Cloning {base_repo_url} ({build_config.branch}) (attempt {attempt}/{max_retries})")
+
+                git.Repo.clone_from(
+                    repo_url,
+                    target_dir,
+                    branch=build_config.branch,
+                    depth=1,
+                    single_branch=True,
+                    env={"GIT_TERMINAL_PROMPT": "0"}  # Prevent git from prompting
+                )
+                self.state.log(f"Cloned to {target_dir}")
+                return target_dir
+
+            except Exception as e:
+                error_msg = mask_token(str(e), token) if token else str(e)
+                self.state.log(f"Clone attempt {attempt} failed: {error_msg}", level="warn")
+
+                # Clean up failed attempt
+                if target_dir.exists():
+                    shutil.rmtree(target_dir, ignore_errors=True)
+
+                if attempt < max_retries:
+                    wait_time = attempt * 2  # 2s, 4s backoff
+                    self.state.log(f"Retrying in {wait_time}s...")
+                    time.sleep(wait_time)
+                else:
+                    self.state.log(f"Clone failed after {max_retries} attempts: {error_msg}", level="error")
+                    raise RuntimeError(f"Clone failed after {max_retries} attempts: {error_msg}")
+
+        # Should never reach here
+        raise RuntimeError("Clone failed unexpectedly")
+
+    def _cleanup_stale_git_locks(self) -> None:
+        """Remove stale git lock files that can cause clone failures."""
+        import glob
+
+        # Clean up any stale .git lock files in temp directories
+        temp_base = tempfile.gettempdir()
+        for lock_file in glob.glob(f"{temp_base}/build_*/.git/index.lock"):
+            try:
+                os.remove(lock_file)
+                self.state.log(f"Removed stale lock: {lock_file}")
+            except Exception:
+                pass
+
+        # Also clean up old temp build directories (older than 1 hour)
+        import time as time_module
+        now = time_module.time()
+        for dir_path in glob.glob(f"{temp_base}/build_*"):
+            try:
+                if now - os.path.getmtime(dir_path) > 3600:  # 1 hour
+                    shutil.rmtree(dir_path, ignore_errors=True)
+            except Exception:
+                pass
 
     def build_and_push(self, build: Build) -> bool:
         # Set trace ID for this build
@@ -1842,19 +1932,47 @@ hivemind = HivemindClient(config, state, builder)
 # Startup
 # =============================================================================
 
+def check_git_available() -> bool:
+    """Check if git is available and working."""
+    try:
+        import git
+        # Try to access git version
+        git.Git().version()
+        return True
+    except Exception as e:
+        state.log(f"Git check failed: {e}", level="error")
+        return False
+
+
 def startup():
     init_telemetry()
 
     state.log(f"HF Builder starting ({config.runner_id})")
     state.log(f"Registry: {config.registry_url}")
 
+    # Check git is available
+    if not check_git_available():
+        state.log("WARNING: git not available - clones may fail", level="error")
+    else:
+        state.log("Git: OK")
+
+    # Setup registry auth
     if config.registry_user:
         builder.setup_registry_auth()
-
+    else:
+        state.log("WARNING: No registry credentials - pushes will fail", level="warn")
+
+    # Log configuration
+    if config.default_image:
+        state.log(f"Default image: {config.default_image}")
+    if config.github_token:
+        state.log(f"GitHub token: configured ({len(config.github_token)} chars)")
+    if config.auto_restart_space:
+        state.log(f"Auto-restart: {config.auto_restart_space}")
     if config.slack_webhook_url:
-        state.log("Slack notifications enabled")
+        state.log("Slack notifications: enabled")
     if config.discord_webhook_url:
-        state.log("Discord notifications enabled")
+        state.log("Discord notifications: enabled")
 
     # Hivemind integration
     if hivemind.enabled:
@@ -1862,6 +1980,9 @@ def startup():
         if hivemind.register():
             threading.Thread(target=hivemind.work_loop, daemon=True).start()
 
+    # Cleanup any stale build directories from previous runs
+    builder._cleanup_stale_git_locks()
+
     state.set_ready(True)
     state.log("Ready")