Claude Code: Fix 'active' state drift with ghost PID detection and subprocess logging

1. Added stdout/stderr logging to subprocess.Popen - writes to /app/logs/brain_*.log
This prevents silent crashes of brain_minimal.py

2. Enhanced _is_process_alive() with 3-tier validation:
- subprocess.poll() to detect exited processes
- psutil.pid_exists() to detect ghost PIDs
- Process name check to verify it's a python process

3. Added heartbeat freshness check to /api/state endpoint:
- If heartbeat > 15s old, forces state to 'idle'
- Prevents stale 'active' states from dead processes
- Returns heartbeat_fresh in response for debugging

Port binding: App already correctly bound to 0.0.0.0:7860 (line 696-697)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

app.py

@@ -115,11 +115,15 @@ class ProcessWatchdog:
             # Ensure logs directory exists
             Path("/app/logs").mkdir(parents=True, exist_ok=True)
 
-            # Spawn subprocess
+            # Open log files for subprocess output (for debugging crashes)
+            stdout_log = open("/app/logs/brain_stdout.log", "a")
+            stderr_log = open("/app/logs/brain_stderr.log", "a")
+
+            # Spawn subprocess with stdout/stderr piped to log files
             self.process = subprocess.Popen(
                 [sys.executable, self.script_path],
-                stdout=subprocess.PIPE,
-                stderr=subprocess.PIPE,
+                stdout=stdout_log,
+                stderr=stderr_log,
                 env=env,
                 cwd="/app",
                 start_new_session=True  # Create process group for clean termination
@@ -181,16 +185,44 @@ class ProcessWatchdog:
         self._log("info", "Monitor thread stopped")
 
     def _is_process_alive(self) -> bool:
-        """Check if the managed process is alive."""
-        if self.process is None:
+        """Check if the managed process is alive and the PID is real."""
+        if self.process is None or self.pid is None:
             return False
 
-        # Try polling the process
+        # First check: try polling the subprocess
         try:
-            return self.process.poll() is None
-        except Exception:
+            poll_result = self.process.poll()
+            if poll_result is not None:
+                # Process has exited
+                self._log("warning", "Process poll returned exit code", exit_code=poll_result, pid=self.pid)
+                return False
+        except Exception as e:
+            self._log("warning", "Process poll failed", error=str(e))
             return False
 
+        # Second check: verify PID exists in system (detect ghost PIDs)
+        try:
+            if not psutil.pid_exists(self.pid):
+                self._log("warning", "Ghost PID detected - process not found in system", pid=self.pid)
+                return False
+        except Exception as e:
+            self._log("warning", "PID check failed", error=str(e))
+
+        # Third check: verify process name matches python (double-verify)
+        try:
+            proc = psutil.Process(self.pid)
+            if "python" not in proc.name().lower():
+                self._log("warning", "PID exists but is not a python process", pid=self.pid, name=proc.name())
+                return False
+        except psutil.NoSuchProcess:
+            self._log("warning", "Process vanished during verification", pid=self.pid)
+            return False
+        except Exception as e:
+            self._log("warning", "Process name check failed", error=str(e))
+
+        # All checks passed
+        return True
+
     def _update_status_file(self):
         """Update cain_status.json with current process info."""
         try:
@@ -449,20 +481,44 @@ async def api_state(request: Request):
     except ImportError:
         status_data = {"current_state": "idle", "stage": "RUNNING", "health": "HEALTHY"}
 
-    watchdog_status = _watchdog.get_status() if _watchdog else {"is_alive": False}
+    watchdog_status = _watchdog.get_status() if _watchdog else {"is_alive": False, "pid": None}
 
-    # Report state: _worker_state (heartbeat) takes precedence over current_state (polling)
-    # This allows heartbeat to show "active" while polling loop uses "idle" for trigger detection
-    reported_state = status_data.get("_worker_state") or status_data.get("current_state", "idle")
+    # Check heartbeat freshness - if heartbeat is older than 15 seconds, worker is likely dead
+    heartbeat_str = status_data.get("worker_heartbeat")
+    heartbeat_fresh = False
+    if heartbeat_str:
+        try:
+            from datetime import timedelta
+            heartbeat_time = datetime.fromisoformat(heartbeat_str.replace("+00:00", "").replace("Z", "+00:00"))
+            if heartbeat_time.endswith("+00:00"):
+                heartbeat_time = heartbeat_time.replace(tzinfo=None)
+            heartbeat_age = (datetime.utcnow() - heartbeat_time.replace(tzinfo=None)).total_seconds()
+            heartbeat_fresh = heartbeat_age < 15
+        except Exception:
+            heartbeat_fresh = False
+
+    # Determine actual state based on watchdog health and heartbeat freshness
+    worker_state = status_data.get("_worker_state", "")
+    current_state = status_data.get("current_state", "idle")
+
+    # If watchdog says process is dead OR heartbeat is stale, force state to idle
+    if not watchdog_status["is_alive"] or not heartbeat_fresh:
+        reported_state = "idle"
+    elif worker_state:
+        # Only trust _worker_state if heartbeat is fresh and process is alive
+        reported_state = worker_state
+    else:
+        reported_state = current_state
 
     return {
         "state": reported_state,
-        "detail": "Cain is operational" if watchdog_status["is_alive"] else "Brain process not running",
+        "detail": "Cain is operational" if watchdog_status["is_alive"] and heartbeat_fresh else "Brain process not running or stale",
         "updated_at": datetime.utcnow().isoformat() + "+00:00",
         "stage": status_data.get("stage", "RUNNING"),
-        "health": "HEALTHY" if watchdog_status["is_alive"] else "UNHEALTHY",
+        "health": "HEALTHY" if watchdog_status["is_alive"] and heartbeat_fresh else "UNHEALTHY",
         "is_alive": watchdog_status["is_alive"],
-        "brain_pid": watchdog_status.get("pid")
+        "brain_pid": watchdog_status.get("pid"),
+        "heartbeat_fresh": heartbeat_fresh
     }