""" Manual testing script for IntegraChat improvements Run this script to test all new features: - Analytics logging - Enhanced admin rules with regex/severity - API endpoints - Agent debug/plan endpoints Usage: python test_manual.py """ import requests import json import time from pathlib import Path import sys # Add backend to path backend_dir = Path(__file__).parent / "backend" sys.path.insert(0, str(backend_dir)) # Also add root for backend.api imports root_dir = Path(__file__).parent sys.path.insert(0, str(root_dir)) BASE_URL = "http://localhost:8000" TENANT_ID = "test_tenant_manual" def print_section(title): print("\n" + "=" * 60) print(f" {title}") print("=" * 60) def test_analytics_store(): """Test AnalyticsStore directly.""" print_section("Testing AnalyticsStore") try: from api.storage.analytics_store import AnalyticsStore store = AnalyticsStore() # Log various events print("Logging tool usage...") store.log_tool_usage(TENANT_ID, "rag", latency_ms=150, tokens_used=500, success=True) store.log_tool_usage(TENANT_ID, "web", latency_ms=80, success=True) store.log_tool_usage(TENANT_ID, "llm", latency_ms=200, tokens_used=1000, success=True) print("Logging red-flag violation...") store.log_redflag_violation( TENANT_ID, "rule1", ".*password.*", "high", "password123", confidence=0.95, message_preview="User asked about password" ) print("Logging RAG search...") store.log_rag_search( TENANT_ID, "What is the company policy?", hits_count=5, avg_score=0.85, top_score=0.92, latency_ms=120 ) print("Logging agent query...") store.log_agent_query( TENANT_ID, "What is the company policy?", intent="rag", tools_used=["rag", "llm"], total_tokens=1000, total_latency_ms=250, success=True ) # Get stats print("\n๐Ÿ“Š Tool Usage Stats:") print(json.dumps(store.get_tool_usage_stats(TENANT_ID), indent=2)) print("\n๐Ÿšจ Red-Flag Violations:") violations = store.get_redflag_violations(TENANT_ID) print(json.dumps(violations, indent=2, default=str)) print("\n๐Ÿ“ˆ Activity Summary:") print(json.dumps(store.get_activity_summary(TENANT_ID), indent=2, default=str)) print("\n๐Ÿ” RAG Quality Metrics:") print(json.dumps(store.get_rag_quality_metrics(TENANT_ID), indent=2)) print("\nโœ… AnalyticsStore tests passed!") return True except Exception as e: print(f"โŒ AnalyticsStore test failed: {e}") import traceback traceback.print_exc() return False def test_admin_rules(): """Test enhanced admin rules with regex and severity.""" print_section("Testing Enhanced Admin Rules") try: from api.storage.rules_store import RulesStore import re store = RulesStore() # Add rules with regex and severity print("Adding rules with regex patterns...") store.add_rule( TENANT_ID, "Block password queries", pattern=".*password.*|.*pwd.*", severity="high", description="Blocks password-related queries" ) store.add_rule( TENANT_ID, "Block email sharing", pattern=".*@.*\\..*", severity="medium", description="Blocks email addresses" ) store.add_rule( TENANT_ID, "Simple keyword rule", severity="low" ) # Get detailed rules rules = store.get_rules_detailed(TENANT_ID) print("\n๐Ÿ“‹ Rules with Metadata:") print(json.dumps(rules, indent=2, default=str)) # Test regex matching print("\n๐Ÿงช Testing Regex Patterns:") for rule in rules: if rule.get("pattern"): pattern = rule["pattern"] regex = re.compile(pattern, re.IGNORECASE) test_cases = [ "What is my password?", "My email is test@example.com", "Just regular text" ] for test_text in test_cases: match = regex.search(test_text) print(f" Pattern: {pattern[:30]}... | Text: \"{test_text}\" | Match: {match is not None}") print("\nโœ… Admin Rules tests passed!") return True except Exception as e: print(f"โŒ Admin Rules test failed: {e}") import traceback traceback.print_exc() return False def test_api_endpoints(): """Test API endpoints.""" print_section("Testing API Endpoints") headers = {"x-tenant-id": TENANT_ID} endpoints = [ ("GET", "/analytics/overview?days=30", None), ("GET", "/analytics/tool-usage?days=30", None), ("GET", "/analytics/rag-quality?days=30", None), ("GET", "/analytics/redflags?limit=50&days=30", None), ("GET", "/admin/rules?detailed=true", None), ("GET", "/admin/violations?limit=50&days=30", None), ("GET", "/admin/tools/logs?days=7", None), ] results = [] for method, endpoint, data in endpoints: try: url = f"{BASE_URL}{endpoint}" if method == "GET": response = requests.get(url, headers=headers, timeout=5) else: response = requests.post(url, headers=headers, json=data, timeout=5) status = "โœ…" if response.status_code == 200 else "โš ๏ธ" print(f"{status} {method} {endpoint} - Status: {response.status_code}") if response.status_code == 200: result = response.json() print(f" Response keys: {list(result.keys())[:5]}") results.append(response.status_code == 200) except requests.exceptions.ConnectionError: print(f"โŒ {method} {endpoint} - Cannot connect to {BASE_URL}") print(" Make sure the FastAPI server is running on port 8000") results.append(False) except Exception as e: print(f"โŒ {method} {endpoint} - Error: {e}") results.append(False) # Test POST endpoints print("\n๐Ÿ“ Testing POST Endpoints...") try: # Add admin rule response = requests.post( f"{BASE_URL}/admin/rules", headers=headers, json={ "rule": "Test rule via API", "pattern": ".*test.*", "severity": "medium" }, timeout=5 ) status = "โœ…" if response.status_code == 200 else "โš ๏ธ" print(f"{status} POST /admin/rules - Status: {response.status_code}") results.append(response.status_code == 200) except Exception as e: print(f"โŒ POST /admin/rules - Error: {e}") results.append(False) # Test agent endpoints (may fail if services not running) print("\n๐Ÿค– Testing Agent Endpoints...") agent_endpoints = [ ("/agent/plan", {"tenant_id": TENANT_ID, "message": "Test message", "temperature": 0.0}), ] for endpoint, data in agent_endpoints: try: response = requests.post( f"{BASE_URL}{endpoint}", json=data, timeout=10 ) status = "โœ…" if response.status_code == 200 else "โš ๏ธ" print(f"{status} POST {endpoint} - Status: {response.status_code}") if response.status_code == 200: result = response.json() print(f" Response keys: {list(result.keys())[:5]}") results.append(response.status_code in [200, 500, 503]) # Accept various status codes except Exception as e: print(f"โš ๏ธ POST {endpoint} - Error: {e} (May be expected if services not running)") results.append(True) # Don't fail if services not running success_count = sum(results) total_count = len(results) print(f"\n๐Ÿ“Š API Endpoint Tests: {success_count}/{total_count} passed") return success_count == total_count or success_count >= total_count * 0.8 # 80% pass rate def main(): """Run all manual tests.""" print("\n" + "๐Ÿš€" * 30) print("IntegraChat Manual Testing Suite") print("๐Ÿš€" * 30) results = [] # Test Analytics Store results.append(test_analytics_store()) time.sleep(1) # Test Admin Rules results.append(test_admin_rules()) time.sleep(1) # Test API Endpoints results.append(test_api_endpoints()) # Summary print_section("Test Summary") passed = sum(results) total = len(results) print(f"Tests Passed: {passed}/{total}") if passed == total: print("โœ… All tests passed!") elif passed >= total * 0.8: print("โš ๏ธ Most tests passed (some may require running services)") else: print("โŒ Some tests failed. Check errors above.") print("\n๐Ÿ’ก Tips:") print(" - For API tests, ensure FastAPI server is running: uvicorn backend.api.main:app --port 8000") print(" - Agent endpoints may require MCP servers and LLM to be running") print(" - Check TESTING_GUIDE.md for more detailed testing instructions") if __name__ == "__main__": main()