Add Agentic Incident Response gradio app

.gitignore

@@ -0,0 +1,4 @@
+.env
+__pycache__/
+*.pyc
+.venv/

README.md

@@ -1,6 +1,6 @@
 ---
 title: Autonomous System Diagnostics
-emoji: 😻
+emoji: 🛰️
 colorFrom: indigo
 colorTo: red
 sdk: gradio
@@ -10,4 +10,20 @@ app_file: app.py
 pinned: false
 ---
 
+# Autonomous System Diagnostics & Remediation
+
+A multi-agent incident response system powered by Gemini. Paste in server logs,
+an incident ticket, and a runbook, and a chain of specialized agents will:
+
+1. **Log Analysis** – surface anomalies and error patterns
+2. **Incident Correlation** – check the logs against the reported ticket
+3. **Root Cause Analysis** – pinpoint the cause using the runbook
+4. **Resolution** – propose a step-by-step remediation plan
+5. **Reporting** – generate a post-incident report (PIR)
+
+## Setup
+
+This Space requires a `GEMINI_API_KEY`. Add it under
+**Settings → Variables and secrets** as a repository secret.
+
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

agents.py

@@ -0,0 +1,92 @@
+import os
+import google.generativeai as genai
+from dotenv import load_dotenv
+
+# Load env variables
+load_dotenv()
+
+class IncidentAgents:
+    def __init__(self):
+        # Configure Gemini
+        api_key = os.getenv("GEMINI_API_KEY")
+        if not api_key:
+            raise ValueError("GEMINI_API_KEY not found in .env file")
+
+        genai.configure(api_key=api_key)
+
+        # We use Gemini 1.5 Flash for speed and efficiency
+        self.model_name = "gemini-2.5-flash"
+
+    def _call_llm(self, role_description, user_content):
+        """Helper to call Gemini API"""
+
+        # Initialize the model with a system instruction (Role)
+        model = genai.GenerativeModel(
+            model_name=self.model_name,
+            system_instruction=role_description
+        )
+
+        # Generate content
+        try:
+            response = model.generate_content(user_content)
+            return response.text
+        except Exception as e:
+            return f"Error communicating with Gemini: {e}"
+
+    # --- AGENT 1: Log Analyst ---
+    def log_analysis_agent(self, logs):
+        print("🔎 Log Analysis Agent Working...")
+        role = """
+        You are a Senior Site Reliability Engineer (SRE).
+        Your task is to analyze server logs.
+        Identify patterns of errors, timestamps of failure start, and specific error messages.
+        Output a concise summary of the anomalies found.
+        """
+        return self._call_llm(role, f"Analyze these logs:\n{logs}")
+
+    # --- AGENT 2: Incident Correlator ---
+    def incident_correlator_agent(self, ticket, log_analysis):
+        print("🔗 Incident Correlator Agent Working...")
+        role = """
+        You are an Incident Commander.
+        Correlate the user-reported incident ticket with the technical log analysis.
+        Confirm if the logs support the ticket description.
+        """
+        content = f"Ticket:\n{ticket}\n\nLog Analysis:\n{log_analysis}"
+        return self._call_llm(role, content)
+
+    # --- AGENT 3: Root Cause Analyst (RCA) ---
+    def root_cause_agent(self, correlation_findings, runbook):
+        print("🧠 Root Cause Agent Working...")
+        role = """
+        You are a Root Cause Analysis Expert.
+        Using the incident correlation and the provided Engineering Runbook, determine the most likely root cause.
+        Cite the specific section of the runbook that matches the symptoms.
+        """
+        content = f"Findings:\n{correlation_findings}\n\nRunbook:\n{runbook}"
+        return self._call_llm(role, content)
+
+    # --- AGENT 4: Resolution Agent ---
+    def resolution_agent(self, root_cause_analysis, runbook):
+        print("🛠️ Resolution Agent Working...")
+        role = """
+        You are a DevOps Automation Engineer.
+        Based on the identified root cause and the runbook, generate a step-by-step remediation plan.
+        If the runbook has specific commands, include them.
+        """
+        content = f"RCA:\n{root_cause_analysis}\n\nRunbook Content:\n{runbook}"
+        return self._call_llm(role, content)
+
+    # --- AGENT 5: Report Generator ---
+    def report_agent(self, ticket, rca, resolution):
+        print("📝 Report Agent Working...")
+        role = """
+        You are a Technical Writer.
+        Generate a professional Post-Incident Report (PIR) in Markdown format.
+        Include:
+        1. Executive Summary
+        2. Root Cause
+        3. Remediation Taken/Suggested
+        """
+        content = f"Ticket: {ticket}\nRCA: {rca}\nResolution: {resolution}"
+        return self._call_llm(role, content)

app.py

@@ -0,0 +1,101 @@
+import gradio as gr
+import os
+import json
+from dotenv import load_dotenv
+from agents import IncidentAgents
+from tools import read_log_file, read_ticket_data, read_runbook
+
+# Load environment variables
+load_dotenv()
+
+# --- DEFAULT DATA (sample incident, used to pre-fill the UI) ---
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+
+DEFAULT_LOGS = read_log_file(os.path.join(BASE_DIR, "data", "logs", "server_logs.txt"))
+DEFAULT_TICKET = json.dumps(
+    read_ticket_data(os.path.join(BASE_DIR, "data", "tickets", "incident_ticket.json")),
+    indent=2
+)
+DEFAULT_RUNBOOK = read_runbook(os.path.join(BASE_DIR, "data", "docs", "runbook.md"))
+
+# --- 2. THE CORE WORKFLOW FUNCTION ---
+def run_incident_response(logs_input, ticket_input, runbook_input):
+    """
+    This function connects the UI inputs to the Agent Logic.
+    """
+    try:
+        # Initialize Agents
+        agents = IncidentAgents()
+
+        # Step A: Log Analysis
+        log_analysis = agents.log_analysis_agent(logs_input)
+        yield log_analysis, "...", "...", "...", "..." # Stream updates to UI
+
+        # Step B: Correlation
+        correlation = agents.incident_correlator_agent(ticket_input, log_analysis)
+        yield log_analysis, correlation, "...", "...", "..."
+
+        # Step C: Root Cause Analysis
+        rca = agents.root_cause_agent(correlation, runbook_input)
+        yield log_analysis, correlation, rca, "...", "..."
+
+        # Step D: Resolution
+        resolution = agents.resolution_agent(rca, runbook_input)
+        yield log_analysis, correlation, rca, resolution, "..."
+
+        # Step E: Final Report
+        final_report = agents.report_agent(ticket_input, rca, resolution)
+        yield log_analysis, correlation, rca, resolution, final_report
+
+    except Exception as e:
+        error_msg = f"Error: {str(e)}"
+        yield error_msg, error_msg, error_msg, error_msg, error_msg
+
+# --- 3. BUILD THE GRADIO UI ---
+with gr.Blocks(title="Agentic Incident Response") as demo:
+
+    # Header
+    gr.Markdown("# AI Agent Incident Response System")
+    gr.Markdown("Enter system logs, an incident ticket, and a runbook. The Multi-Agent System will analyze, correlate, and fix the issue.")
+
+    # Input Section
+    with gr.Row():
+        with gr.Column():
+            logs_box = gr.Textbox(label="1. System Logs", value=DEFAULT_LOGS, lines=8, placeholder="Paste logs here...")
+        with gr.Column():
+            ticket_box = gr.Textbox(label="2. Incident Ticket (JSON)", value=DEFAULT_TICKET, lines=8, placeholder="Paste ticket JSON here...")
+        with gr.Column():
+            runbook_box = gr.Textbox(label="3. Runbook (Markdown)", value=DEFAULT_RUNBOOK, lines=8, placeholder="Paste runbook here...")
+
+    # Action Button
+    run_btn = gr.Button("Run Automated Response Workflow", variant="primary")
+
+    # Output Section (The "Agent Minds")
+    gr.Markdown("### Agent Reasoning Chain")
+
+    with gr.Accordion("Step 1: Log Analysis Agent", open=False):
+        out_logs = gr.Markdown()
+
+    with gr.Accordion("Step 2: Incident Correlator Agent", open=False):
+        out_correlation = gr.Markdown()
+
+    with gr.Accordion("Step 3: Root Cause Analysis (RCA) Agent", open=False):
+        out_rca = gr.Markdown()
+
+    with gr.Accordion("Step 4: Resolution Agent", open=True):
+        out_resolution = gr.Markdown()
+
+    # Final Report Section
+    gr.Markdown("### Final Post-Incident Report")
+    out_final_report = gr.Markdown()
+
+    # Click Event
+    run_btn.click(
+        fn=run_incident_response,
+        inputs=[logs_box, ticket_box, runbook_box],
+        outputs=[out_logs, out_correlation, out_rca, out_resolution, out_final_report]
+    )
+
+# Launch
+if __name__ == "__main__":
+    demo.launch(theme=gr.themes.Soft())

data/docs/runbook.md

@@ -0,0 +1,16 @@
+# Engineering Runbook: Database Connection Issues
+
+## Error: ConnectionPoolTimeoutError
+**Symptoms:** - API returns 500 status codes.
+- Logs show "Timeout waiting for connection from pool".
+- Latency spikes on endpoints /api/v1/users.
+
+## Root Causes
+1. **Traffic Spike:** Sudden increase in concurrent users exceeding `MAX_CONNECTIONS`.
+2. **Zombie Connections:** Application not closing connections properly (missing `.close()` or `try/finally` blocks).
+3. **Database Maintenance:** DB is undergoing patches or backups.
+
+## Remediation Steps
+1. **Immediate Mitigation:** Restart the application service (`sudo systemctl restart api-service`) to flush the pool.
+2. **Investigation:** Check `pg_stat_activity` for idle transactions.
+3. **Long-term Fix:** Increase `MAX_CONNECTIONS` in `config.yaml` or implement connection pooling proxy (PgBouncer).

data/logs/server_logs.txt

@@ -0,0 +1,8 @@
+2024-10-24 14:28:10 [INFO] Request received: GET /api/v1/health - 200 OK
+2024-10-24 14:29:15 [INFO] Worker process started (PID 4021)
+2024-10-24 14:30:05 [WARN] Connection pool usage at 85%
+2024-10-24 14:30:45 [WARN] Connection pool usage at 95% - approaching limit
+2024-10-24 14:31:02 [ERROR] [PID 4021] Failed to acquire connection. Error: sqlalchemy.exc.TimeoutError: QueuePool limit of size 10 overflow 10 reached, connection timed out, timeout 30.
+2024-10-24 14:31:02 [ERROR] Request failed: GET /api/v1/users/login - 500 Internal Server Error
+2024-10-24 14:31:05 [ERROR] [PID 4021] Failed to acquire connection.
+2024-10-24 14:31:10 [CRITICAL] Service health check failing. Database unreachable.

data/tickets/incident_ticket.json

@@ -0,0 +1,8 @@
+{
+  "ticket_id": "INC-2024-001",
+  "priority": "P1",
+  "status": "OPEN",
+  "title": "API returning 500 errors on User Service",
+  "description": "Customer reports unable to login. Monitoring dashboard shows 500 error rate spike starting at 14:30 UTC.",
+  "reported_at": "2024-10-24T14:35:00Z"
+}

requirements.txt

@@ -0,0 +1,3 @@
+google-generativeai
+python-dotenv
+gradio

tools.py

@@ -0,0 +1,26 @@
+import os
+import json
+
+def read_log_file(filepath):
+    """Reads the raw log file."""
+    try:
+        with open(filepath, 'r') as f:
+            return f.read()
+    except FileNotFoundError:
+        return "Error: Log file not found."
+
+def read_ticket_data(filepath):
+    """Reads the JSON incident ticket."""
+    try:
+        with open(filepath, 'r') as f:
+            return json.load(f)
+    except FileNotFoundError:
+        return {"error": "Ticket file not found."}
+
+def read_runbook(filepath):
+    """Reads the markdown runbook."""
+    try:
+        with open(filepath, 'r') as f:
+            return f.read()
+    except FileNotFoundError:
+        return "Error: Runbook not found."