Implement PPID orphan watchdog, tini zombie reaping, connection pooling caps, point-in-time zip snapshots, and db keep-alive heartbeats

Dockerfile

@@ -2,7 +2,7 @@ FROM python:3.11-slim
 
 # Install system dependencies
 RUN apt-get update && apt-get install -y --no-install-recommends \
-    git bash curl && \
+    git bash curl tini && \
     rm -rf /var/lib/apt/lists/*
 
 # Create non-root user (HF requirement: uid 1000)
@@ -25,4 +25,5 @@ RUN mkdir -p /tmp/workspace
 
 EXPOSE 7860
 
+ENTRYPOINT ["tini", "--"]
 CMD ["python", "backend.py"]

backend.py

@@ -355,7 +355,13 @@ async def init_db():
     if not DATABASE_URL:
         return
     try:
-        db_pool = await asyncpg.create_pool(DATABASE_URL, ssl="require", min_size=1, max_size=5)
+        db_pool = await asyncpg.create_pool(
+            DATABASE_URL, 
+            ssl="require", 
+            min_size=1, 
+            max_size=3,
+            max_inactive_connection_lifetime=300
+        )
         async with db_pool.acquire() as conn:
             await conn.execute("""
                 CREATE TABLE IF NOT EXISTS agent_sessions (
@@ -1208,20 +1214,35 @@ from fastapi.responses import FileResponse
 @app.get("/api/backup/download")
 async def download_backup(authorization: str = Header(None)):
     auth(authorization)
+    snapshot_dir = "/tmp/workspace_snapshot"
     archive_base = "/tmp/workspace_backup_download"
     archive_zip = archive_base + ".zip"
-    if os.path.exists(archive_zip):
-        try:
-            os.unlink(archive_zip)
-        except Exception:
-            pass
-            
+    
+    # Clean up old files/folders
+    for path in [snapshot_dir, archive_zip]:
+        if os.path.exists(path):
+            try:
+                if os.path.isdir(path):
+                    shutil.rmtree(path)
+                else:
+                    os.unlink(path)
+            except Exception:
+                pass
+                
     try:
-        shutil.make_archive(archive_base, 'zip', WORKSPACE_DIR)
+        # 1. Atomic-like snapshot copy (ignoring temporary files)
+        shutil.copytree(WORKSPACE_DIR, snapshot_dir, symlinks=True, ignore=shutil.ignore_patterns('.git', 'node_modules', '.next'))
+        # 2. Archive the snapshot folder to disk to prevent OOM memory spike
+        shutil.make_archive(archive_base, 'zip', snapshot_dir)
+        # 3. Clean up the snapshot directory immediately
+        shutil.rmtree(snapshot_dir)
+        
         if not os.path.exists(archive_zip):
             raise HTTPException(status_code=500, detail="Failed to create zip archive")
         return FileResponse(archive_zip, media_type="application/zip", filename="workspace_backup.zip")
     except Exception as e:
+        if os.path.exists(snapshot_dir):
+            shutil.rmtree(snapshot_dir)
         raise HTTPException(status_code=500, detail=str(e))
 
 
@@ -1239,38 +1260,66 @@ async def health():
 
 
 # ---------------------------------------------------------------------------
-# Watchdog Daemon for Claude Code Subprocesses
+# Watchdog Daemon for Claude Code Subprocesses (Orphan Reaper)
 # ---------------------------------------------------------------------------
 
 def run_watchdog():
-    log_activity("System Watchdog Daemon started")
+    log_activity("System Watchdog Daemon started (PPID-based Orphan detection)")
     while True:
         try:
             import psutil
-            for proc in psutil.process_iter(['pid', 'name', 'cmdline', 'create_time']):
+            for proc in psutil.process_iter(['pid', 'ppid', 'name', 'cmdline', 'status']):
                 try:
                     cmd = " ".join(proc.info['cmdline'] or [])
-                    if "claude-code" in cmd.lower() or "anthropic" in cmd.lower():
-                        elapsed = time.time() - proc.info['create_time']
-                        if elapsed > 600: # 10 minutes limit
-                            log_activity(f"[Watchdog SIGKILL] Reaping hung Claude Code process PID {proc.info['pid']} (Active for {elapsed:.1f}s)")
-                            proc.kill()
+                    # Match the CLI binary (looks like claude-code or anthropic CLI wrapper)
+                    if "claude" in cmd.lower() or "anthropic" in cmd.lower():
+                        ppid = proc.info['ppid']
+                        pid = proc.info['pid']
+                        
+                        # Is the parent still alive and not a zombie?
+                        parent_exists = False
+                        if ppid != 1:  # Orphaned processes get reparented to PID 1 in Linux
+                            try:
+                                parent_proc = psutil.Process(ppid)
+                                if parent_proc.is_running() and parent_proc.status() != psutil.STATUS_ZOMBIE:
+                                    parent_exists = True
+                            except psutil.NoSuchProcess:
+                                pass
+                        
+                        if not parent_exists:
+                            log_activity(f"[Watchdog SIGKILL] Reaping orphaned Claude Code process PID {pid} (PPID {ppid})")
+                            proc.terminate()
+                            time.sleep(2)
+                            if proc.is_running():
+                                proc.kill()
                 except Exception:
                     continue
         except ImportError:
-            # Fallback zero-dependency shell parser
+            # Fallback zero-dependency shell parser using /proc
             try:
-                out = subprocess.check_output("ps -o pid,etime,args | grep -E 'claude-code|anthropic' | grep -v grep", shell=True, text=True)
+                # Find all processes and examine their parent PID
+                out = subprocess.check_output("ps -o pid,ppid,args | grep -E 'claude|anthropic' | grep -v grep", shell=True, text=True)
                 for line in out.strip().split("\n"):
                     parts = line.strip().split(None, 2)
                     if len(parts) >= 2:
                         pid = int(parts[0])
-                        etime = parts[1]
-                        # Check if running > 10 mins (format dd-hh:mm:ss or mm:ss)
-                        is_stale = "-" in etime or len(etime.split(":")) > 2 or (len(etime.split(":")) == 2 and int(etime.split(":")[0]) > 10)
-                        if is_stale:
-                            log_activity(f"[Watchdog SIGKILL Fallback] Reaping hung process PID {pid} (etime: {etime})")
-                            os.kill(pid, signal.SIGKILL)
+                        ppid = int(parts[1])
+                        
+                        # Check if parent pid exists/is alive
+                        parent_exists = False
+                        if ppid != 1:
+                            # Check /proc/[ppid] directory
+                            if os.path.exists(f"/proc/{ppid}"):
+                                parent_exists = True
+                                
+                        if not parent_exists:
+                            log_activity(f"[Watchdog SIGKILL Fallback] Reaping orphaned process PID {pid} (PPID {ppid})")
+                            try:
+                                os.kill(pid, signal.SIGTERM)
+                                time.sleep(2)
+                                os.kill(pid, signal.SIGKILL)
+                            except Exception:
+                                pass
             except Exception:
                 pass
         except Exception as e:
@@ -1278,10 +1327,25 @@ def run_watchdog():
         time.sleep(60)
 
 
+async def db_heartbeat_loop():
+    log_activity("Database Heartbeat task started")
+    while True:
+        try:
+            if db_pool:
+                async with db_pool.acquire() as conn:
+                    await conn.execute("SELECT 1")
+                    log_activity("[Heartbeat] Pinged Aiven PostgreSQL successfully")
+        except Exception as e:
+            log_activity(f"[Heartbeat Warning] Failed to ping database: {e}")
+        await asyncio.sleep(240) # Every 4 minutes
+
+
 @app.on_event("startup")
 async def startup_event():
     # Start the watchdog thread on startup
     threading.Thread(target=run_watchdog, daemon=True).start()
+    # Start the db keep-alive loop on FastAPI event loop
+    asyncio.create_task(db_heartbeat_loop())
 
 
 # ---------------------------------------------------------------------------

requirements.txt

@@ -3,3 +3,5 @@ uvicorn[standard]==0.34.2
 openai==1.86.0
 asyncpg==0.30.0
 anyio==4.9.0
+psutil==5.9.8
+