commit streamlit

Makefile

@@ -35,6 +35,7 @@ health:
 	@docker-compose exec postgres pg_isready -U postgres > /dev/null 2>&1 && echo "✅ PostgreSQL: Healthy" || echo "❌ PostgreSQL: Unhealthy"
 	@curl -s http://localhost:8000/health > /dev/null && echo "✅ MCP Server: Healthy" || echo "❌ MCP Server: Unhealthy"
 	@curl -s http://localhost:3000 > /dev/null && echo "✅ Frontend: Healthy" || echo "❌ Frontend: Unhealthy"
+	@curl -s http://localhost:8501 > /dev/null && echo "✅ Streamlit: Healthy" || echo "❌ Streamlit: Unhealthy"
 	@docker-compose ps agent | grep -q "Up" && echo "✅ Agent: Running" || echo "❌ Agent: Not running"
 
 # Run integration test
@@ -52,10 +53,11 @@ demo:
 	@make seed
 	@echo ""
 	@echo "🎉 Demo Ready!"
-	@echo "1. Open http://localhost:3000 in your browser"
-	@echo "2. Ask a question like: 'Show me all customers who have placed orders'"
-	@echo "3. Watch the agent process through the workflow"
-	@echo "4. Check Neo4j Browser at http://localhost:7474 (neo4j/password)"
+	@echo "1. Open http://localhost:3000 in your browser (Main Chat Interface)"
+	@echo "2. Open http://localhost:8501 in your browser (Streamlit Monitor)"
+	@echo "3. Ask a question like: 'Show me all customers who have placed orders'"
+	@echo "4. Watch the agent process through the workflow"
+	@echo "5. Check Neo4j Browser at http://localhost:7474 (neo4j/password)"
 	@echo ""
 	@echo "During 5-minute pauses, you can edit instructions in Neo4j Browser:"
 	@echo "MATCH (i:Instruction {status: 'pending'}) SET i.parameters = '{\"question\": \"new question\"}'"
@@ -74,6 +76,9 @@ restart-mcp:
 restart-frontend:
 	docker-compose restart frontend
 
+restart-streamlit:
+	docker-compose restart streamlit
+
 # Debug commands
 debug-agent:
 	docker-compose logs agent
@@ -84,6 +89,9 @@ debug-mcp:
 debug-frontend:
 	docker-compose logs frontend
 
+debug-streamlit:
+	docker-compose logs streamlit
+
 # Quick status check
 status:
 	docker-compose ps

SYSTEM_OVERVIEW.md

@@ -1,3 +1,8 @@
+#### Quick start prompt:  
+Scan the repo: list the directory tree, key config files, and required env vars. Summarize how to start the system using existing scripts. Do not modify files; just report and wait for confirmation before any changes. 
+Proceed with a fresh start by leveraging: powershell -ExecutionPolicy Bypass -File ops/scripts/fresh_start.ps1
+
+
 # Graph-Driven Agentic System with Human-in-the-Loop Controls
 
 ## What This System Is

docker-compose.yml

@@ -101,6 +101,21 @@ services:
     networks:
       - agent-network
 
+  streamlit:
+    build: ./streamlit
+    ports:
+      - "8501:8501"
+    environment:
+      - MCP_URL=http://mcp:8000/mcp
+      - MCP_API_KEY=dev-key-123
+    depends_on:
+      mcp:
+        condition: service_healthy
+    volumes:
+      - ./streamlit:/app
+    networks:
+      - agent-network
+
 networks:
   agent-network:
     driver: bridge

streamlit/Dockerfile

@@ -0,0 +1,19 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+# Install system dependencies
+RUN apt-get update && apt-get install -y curl && rm -rf /var/lib/apt/lists/*
+
+# Copy requirements and install Python dependencies
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy application code
+COPY . .
+
+# Expose port
+EXPOSE 8501
+
+# Run Streamlit
+CMD ["streamlit", "run", "app.py", "--server.address=0.0.0.0"]

streamlit/app.py

@@ -0,0 +1,484 @@
+#!/usr/bin/env python3
+"""
+Streamlit MCP Monitor & Query Tester
+A lightweight monitoring and testing interface for the agentic system.
+All database access MUST go through MCP server - no direct connections allowed.
+"""
+
+import streamlit as st
+import requests
+import time
+import json
+import pandas as pd
+from datetime import datetime, timedelta
+import os
+from typing import Dict, Any, Optional, Tuple
+
+# Configuration
+MCP_URL = os.getenv("MCP_URL", "http://mcp:8000/mcp")
+MCP_API_KEY = os.getenv("MCP_API_KEY", "dev-key-123")
+
+# Page configuration
+st.set_page_config(
+    page_title="MCP Monitor & Query Tester",
+    page_icon="🤖",
+    layout="wide",
+    initial_sidebar_state="expanded"
+)
+
+# Initialize session state
+if 'workflow_id' not in st.session_state:
+    st.session_state.workflow_id = None
+if 'debug_log' not in st.session_state:
+    st.session_state.debug_log = []
+if 'last_refresh' not in st.session_state:
+    st.session_state.last_refresh = None
+
+def call_mcp(tool: str, params: Optional[Dict[str, Any]] = None) -> Tuple[Dict[str, Any], int]:
+    """
+    Call MCP server - the ONLY way to access databases.
+    Returns (response_data, response_time_ms)
+    """
+    start_time = time.time()
+    
+    try:
+        response = requests.post(
+            MCP_URL,
+            headers={
+                "X-API-Key": MCP_API_KEY,
+                "Content-Type": "application/json"
+            },
+            json={"tool": tool, "params": params or {}},
+            timeout=10
+        )
+        
+        response_time = int((time.time() - start_time) * 1000)
+        
+        # Log the request/response for debugging
+        debug_entry = {
+            "timestamp": datetime.now().isoformat(),
+            "tool": tool,
+            "params": params,
+            "status_code": response.status_code,
+            "response_time_ms": response_time,
+            "success": response.status_code == 200
+        }
+        st.session_state.debug_log.append(debug_entry)
+        
+        # Keep only last 5 entries
+        if len(st.session_state.debug_log) > 5:
+            st.session_state.debug_log = st.session_state.debug_log[-5:]
+        
+        if response.status_code == 200:
+            return response.json(), response_time
+        else:
+            return {"error": f"HTTP {response.status_code}: {response.text}"}, response_time
+            
+    except requests.exceptions.RequestException as e:
+        response_time = int((time.time() - start_time) * 1000)
+        error_msg = f"MCP Server Error: {str(e)}"
+        
+        # Log the error
+        debug_entry = {
+            "timestamp": datetime.now().isoformat(),
+            "tool": tool,
+            "params": params,
+            "status_code": 0,
+            "response_time_ms": response_time,
+            "success": False,
+            "error": error_msg
+        }
+        st.session_state.debug_log.append(debug_entry)
+        
+        return {"error": error_msg}, response_time
+
+def test_neo4j_connection() -> Tuple[bool, int, str]:
+    """Test Neo4j connection through MCP server"""
+    result, response_time = call_mcp("get_schema")
+    if "error" in result:
+        return False, response_time, result["error"]
+    return True, response_time, "Connected"
+
+def test_postgres_connection() -> Tuple[bool, int, str]:
+    """Test PostgreSQL connection through MCP server"""
+    result, response_time = call_mcp("query_postgres", {"query": "SELECT 1 as test"})
+    if "error" in result:
+        return False, response_time, result["error"]
+    return True, response_time, "Connected"
+
+def test_mcp_server() -> Tuple[bool, int, str]:
+    """Test MCP server health"""
+    try:
+        start_time = time.time()
+        response = requests.get(f"{MCP_URL.replace('/mcp', '/health')}", timeout=5)
+        response_time = int((time.time() - start_time) * 1000)
+        
+        if response.status_code == 200:
+            return True, response_time, "Healthy"
+        else:
+            return False, response_time, f"HTTP {response.status_code}"
+    except Exception as e:
+        return False, 0, str(e)
+
+def get_performance_stats() -> Dict[str, Any]:
+    """Get performance statistics through MCP"""
+    result, _ = call_mcp("query_graph", {
+        "query": "MATCH (l:Log) WHERE l.timestamp > datetime() - duration('PT1H') RETURN count(l) as count"
+    })
+    
+    if "error" in result:
+        return {"error": result["error"]}
+    
+    return result.get("data", [{}])[0] if result.get("data") else {}
+
+def create_workflow(question: str) -> Optional[str]:
+    """Create a new workflow for the given question"""
+    workflow_id = f"streamlit-{int(time.time())}"
+    
+    # Create workflow node
+    workflow_result, _ = call_mcp("write_graph", {
+        "action": "create_node",
+        "label": "Workflow",
+        "properties": {
+            "id": workflow_id,
+            "name": f"Streamlit Query: {question[:50]}...",
+            "description": f"Query from Streamlit: {question}",
+            "status": "active",
+            "created_at": datetime.now().isoformat(),
+            "source": "streamlit"
+        }
+    })
+    
+    if "error" in workflow_result:
+        st.error(f"Failed to create workflow: {workflow_result['error']}")
+        return None
+    
+    # Create instruction sequence
+    instructions = [
+        {
+            "id": f"{workflow_id}-inst-1",
+            "type": "discover_schema",
+            "sequence": 1,
+            "description": "Discover database schema",
+            "status": "pending",
+            "pause_duration": 5,  # Short pause for testing
+            "parameters": "{}"
+        },
+        {
+            "id": f"{workflow_id}-inst-2",
+            "type": "generate_sql",
+            "sequence": 2,
+            "description": f"Generate SQL for: {question}",
+            "status": "pending",
+            "pause_duration": 5,
+            "parameters": json.dumps({"question": question})
+        },
+        {
+            "id": f"{workflow_id}-inst-3",
+            "type": "review_results",
+            "sequence": 3,
+            "description": "Review and format results",
+            "status": "pending",
+            "pause_duration": 0,
+            "parameters": "{}"
+        }
+    ]
+    
+    # Create instruction nodes
+    for inst in instructions:
+        inst_result, _ = call_mcp("write_graph", {
+            "action": "create_node",
+            "label": "Instruction",
+            "properties": inst
+        })
+        
+        if "error" in inst_result:
+            st.error(f"Failed to create instruction: {inst_result['error']}")
+            return None
+        
+        # Link instruction to workflow
+        link_result, _ = call_mcp("query_graph", {
+            "query": """
+                MATCH (w:Workflow {id: $workflow_id}), (i:Instruction {id: $inst_id})
+                CREATE (w)-[:HAS_INSTRUCTION]->(i)
+            """,
+            "parameters": {"workflow_id": workflow_id, "inst_id": inst["id"]}
+        })
+    
+    # Create instruction chain
+    for i in range(len(instructions) - 1):
+        chain_result, _ = call_mcp("query_graph", {
+            "query": """
+                MATCH (i1:Instruction {id: $id1}), (i2:Instruction {id: $id2})
+                CREATE (i1)-[:NEXT_INSTRUCTION]->(i2)
+            """,
+            "parameters": {"id1": instructions[i]["id"], "id2": instructions[i + 1]["id"]}
+        })
+    
+    return workflow_id
+
+def get_workflow_status(workflow_id: str) -> Dict[str, Any]:
+    """Get workflow execution status"""
+    result, _ = call_mcp("query_graph", {
+        "query": """
+            MATCH (w:Workflow {id: $id})-[:HAS_INSTRUCTION]->(i:Instruction)
+            RETURN w.status as workflow_status, 
+                   collect(i.status) as instruction_statuses,
+                   collect(i.type) as instruction_types,
+                   collect(i.sequence) as sequences
+        """,
+        "parameters": {"id": workflow_id}
+    })
+    
+    if "error" in result or not result.get("data"):
+        return {"error": "Workflow not found"}
+    
+    return result["data"][0]
+
+def get_workflow_results(workflow_id: str) -> Dict[str, Any]:
+    """Get workflow execution results"""
+    result, _ = call_mcp("query_graph", {
+        "query": """
+            MATCH (w:Workflow {id: $id})-[:HAS_INSTRUCTION]->(i:Instruction)-[:EXECUTED_AS]->(e:Execution)
+            RETURN i.sequence as sequence,
+                   i.type as type,
+                   i.description as description,
+                   e.result as result,
+                   e.started_at as started_at,
+                   e.completed_at as completed_at
+            ORDER BY i.sequence
+        """,
+        "parameters": {"id": workflow_id}
+    })
+    
+    if "error" in result:
+        return {"error": result["error"]}
+    
+    return {"executions": result.get("data", [])}
+
+def get_schema_context() -> str:
+    """Get database schema context for display"""
+    result, _ = call_mcp("query_graph", {
+        "query": """
+            MATCH (t:Table)-[:HAS_COLUMN]->(c:Column)
+            RETURN t.name as table_name, 
+                   collect({name: c.name, type: c.data_type, nullable: c.nullable}) as columns
+            ORDER BY t.name
+        """
+    })
+    
+    if "error" in result:
+        return f"Error fetching schema: {result['error']}"
+    
+    schema_text = "Database Schema:\n"
+    for record in result.get("data", []):
+        table_name = record["table_name"]
+        columns = record["columns"]
+        schema_text += f"\nTable: {table_name}\n"
+        for col in columns:
+            nullable = "NULL" if col["nullable"] else "NOT NULL"
+            schema_text += f"  - {col['name']}: {col['type']} {nullable}\n"
+    
+    return schema_text
+
+def main():
+    st.title("🤖 MCP Monitor & Query Tester")
+    st.caption("Monitor agentic system health and test queries through MCP server")
+    
+    # Sidebar
+    with st.sidebar:
+        st.header("🔧 Configuration")
+        st.code(f"MCP URL: {MCP_URL}")
+        st.code(f"API Key: {MCP_API_KEY[:10]}...")
+        
+        if st.button("🔄 Refresh All", type="primary"):
+            st.rerun()
+        
+        st.header("📊 Quick Stats")
+        stats = get_performance_stats()
+        if "error" not in stats:
+            st.metric("Logs (1h)", stats.get("count", 0))
+        else:
+            st.error(f"Stats error: {stats['error']}")
+    
+    # Main tabs
+    tab1, tab2 = st.tabs(["🔌 Connection Status", "🤖 Query Tester"])
+    
+    with tab1:
+        st.header("Connection Status Monitor")
+        st.caption("All database access goes through MCP server - no direct connections allowed")
+        
+        # Connection status in columns
+        col1, col2, col3 = st.columns(3)
+        
+        with col1:
+            st.subheader("Neo4j (via MCP)")
+            neo4j_ok, neo4j_time, neo4j_msg = test_neo4j_connection()
+            st.metric(
+                label="Status",
+                value="Online" if neo4j_ok else "Offline",
+                delta=f"{neo4j_time}ms"
+            )
+            if neo4j_ok:
+                st.success(neo4j_msg)
+            else:
+                st.error(neo4j_msg)
+        
+        with col2:
+            st.subheader("PostgreSQL (via MCP)")
+            postgres_ok, postgres_time, postgres_msg = test_postgres_connection()
+            st.metric(
+                label="Status",
+                value="Online" if postgres_ok else "Offline",
+                delta=f"{postgres_time}ms"
+            )
+            if postgres_ok:
+                st.success(postgres_msg)
+            else:
+                st.error(postgres_msg)
+        
+        with col3:
+            st.subheader("MCP Server")
+            mcp_ok, mcp_time, mcp_msg = test_mcp_server()
+            st.metric(
+                label="Status",
+                value="Online" if mcp_ok else "Offline",
+                delta=f"{mcp_time}ms"
+            )
+            if mcp_ok:
+                st.success(mcp_msg)
+            else:
+                st.error(mcp_msg)
+        
+        # Performance stats
+        st.subheader("Performance Statistics")
+        stats = get_performance_stats()
+        if "error" not in stats:
+            st.info(f"Operations in last hour: {stats.get('count', 0)}")
+        else:
+            st.error(f"Cannot fetch stats: {stats['error']}")
+        
+        # Auto-refresh info
+        st.session_state.last_refresh = datetime.now()
+        st.caption(f"Last checked: {st.session_state.last_refresh.strftime('%H:%M:%S')}")
+        
+        # Auto-refresh every 5 seconds
+        time.sleep(5)
+        st.rerun()
+    
+    with tab2:
+        st.header("Query Tester")
+        st.caption("Test natural language queries through the agentic engine")
+        
+        # Query input
+        question = st.text_area(
+            "Enter your question:",
+            height=100,
+            placeholder="e.g., 'How many customers do we have?' or 'Show me all orders from last month'"
+        )
+        
+        col1, col2 = st.columns([1, 1])
+        
+        with col1:
+            if st.button("🚀 Execute Query", type="primary", disabled=not question.strip()):
+                if question.strip():
+                    with st.spinner("Creating workflow..."):
+                        workflow_id = create_workflow(question.strip())
+                        if workflow_id:
+                            st.session_state.workflow_id = workflow_id
+                            st.success(f"Workflow created: {workflow_id}")
+                        else:
+                            st.error("Failed to create workflow")
+        
+        with col2:
+            if st.button("🗑️ Clear Results"):
+                st.session_state.workflow_id = None
+                st.rerun()
+        
+        # Workflow execution monitoring
+        if st.session_state.workflow_id:
+            st.subheader("Execution Progress")
+            
+            # Get workflow status
+            status = get_workflow_status(st.session_state.workflow_id)
+            
+            if "error" in status:
+                st.error(f"Status error: {status['error']}")
+            else:
+                workflow_status = status.get("workflow_status", "unknown")
+                instruction_statuses = status.get("instruction_statuses", [])
+                instruction_types = status.get("instruction_types", [])
+                
+                # Progress bar
+                completed = sum(1 for s in instruction_statuses if s == "complete")
+                total = len(instruction_statuses)
+                progress = completed / total if total > 0 else 0
+                
+                st.progress(progress)
+                st.caption(f"Progress: {completed}/{total} instructions completed")
+                
+                # Status display
+                status_cols = st.columns(len(instruction_types))
+                for i, (inst_type, inst_status) in enumerate(zip(instruction_types, instruction_statuses)):
+                    with status_cols[i]:
+                        if inst_status == "complete":
+                            st.success(f"✅ {inst_type}")
+                        elif inst_status == "executing":
+                            st.warning(f"🔄 {inst_type}")
+                        elif inst_status == "failed":
+                            st.error(f"❌ {inst_type}")
+                        else:
+                            st.info(f"⏳ {inst_type}")
+                
+                # Get and display results
+                if completed > 0:
+                    results = get_workflow_results(st.session_state.workflow_id)
+                    
+                    if "error" not in results:
+                        st.subheader("Execution Results")
+                        
+                        for execution in results.get("executions", []):
+                            with st.expander(f"Step {execution['sequence']}: {execution['type']}"):
+                                st.write(f"**Description:** {execution['description']}")
+                                
+                                if execution['started_at'] and execution['completed_at']:
+                                    start = datetime.fromisoformat(execution['started_at'].replace('Z', '+00:00'))
+                                    end = datetime.fromisoformat(execution['completed_at'].replace('Z', '+00:00'))
+                                    duration = (end - start).total_seconds()
+                                    st.write(f"**Duration:** {duration:.2f} seconds")
+                                
+                                if execution['result']:
+                                    try:
+                                        result_data = json.loads(execution['result']) if isinstance(execution['result'], str) else execution['result']
+                                        
+                                        if execution['type'] == 'generate_sql' and 'generated_sql' in result_data:
+                                            st.write("**Generated SQL:**")
+                                            st.code(result_data['generated_sql'], language='sql')
+                                        
+                                        if 'data' in result_data and result_data['data']:
+                                            st.write("**Query Results:**")
+                                            df = pd.DataFrame(result_data['data'])
+                                            st.dataframe(df)
+                                        
+                                        if 'error' in result_data:
+                                            st.error(f"Error: {result_data['error']}")
+                                            
+                                    except Exception as e:
+                                        st.write("**Raw Result:**")
+                                        st.code(str(execution['result']))
+                    else:
+                        st.error(f"Results error: {results['error']}")
+        
+        # Debug information
+        with st.expander("🔧 Debug Information"):
+            st.write("**Last 5 MCP Requests:**")
+            for entry in st.session_state.debug_log:
+                status_icon = "✅" if entry["success"] else "❌"
+                st.write(f"{status_icon} {entry['timestamp']} - {entry['tool']} ({entry['response_time_ms']}ms)")
+                if not entry["success"] and "error" in entry:
+                    st.error(f"Error: {entry['error']}")
+            
+            st.write("**Important:** All database operations go through MCP server. Direct database access is not permitted.")
+
+if __name__ == "__main__":
+    main()

streamlit/requirements.txt

@@ -0,0 +1,4 @@
+streamlit==1.28.0
+requests==2.31.0
+pandas==2.1.0
+python-dotenv==1.0.0