Upload core/performance_monitoring.py with huggingface_hub

core/performance_monitoring.py

@@ -1,268 +1,539 @@
-# backend/core/performance_monitoring.py
+"""
+Performance Baselines and Regression Detection System
+Automated performance monitoring with baseline establishment and regression detection
+"""
 
 import asyncio
-import logging
-import time
-from collections import defaultdict
-from dataclasses import dataclass
-from datetime import UTC, datetime
-from typing import Any
+import json
+import os
+import statistics
+from datetime import datetime, timedelta
+from typing import Dict, List
 
-logger = logging.getLogger(__name__)
+import aiohttp
+import asyncpg
 
+# Simplified version without scipy dependency
+try:
+    import numpy as np
 
-@dataclass
-class PerformanceMetrics:
-    """Comprehensive performance metrics collection"""
+    HAS_NUMPY = True
+except ImportError:
+    HAS_NUMPY = False
+    import statistics
 
-    request_count: int = 0
-    total_response_time: float = 0.0
-    average_response_time: float = 0.0
-    min_response_time: float = float("inf")
-    max_response_time: float = 0.0
-    p95_response_time: float = 0.0
-    p99_response_time: float = 0.0
+from core.config import settings
+from core.logging import logger
 
-    error_count: int = 0
-    error_rate: float = 0.0
 
-    slow_requests_count: int = 0
-    slow_requests_threshold: float = 1.0  # seconds
+class PerformanceMetrics:
+    """Performance metrics container"""
+
+    def __init__(self):
+        self.response_time = 0.0
+        self.throughput = 0.0
+        self.error_rate = 0.0
+        self.cpu_usage = 0.0
+        self.memory_usage = 0.0
+        self.database_query_time = 0.0
+        self.cache_hit_rate = 0.0
+        self.timestamp = datetime.now()
+
+    def to_dict(self) -> Dict:
+        return {
+            "response_time_ms": self.response_time,
+            "throughput_rps": self.throughput,
+            "error_rate_percent": self.error_rate,
+            "cpu_usage_percent": self.cpu_usage,
+            "memory_usage_percent": self.memory_usage,
+            "database_query_time_ms": self.database_query_time,
+            "cache_hit_rate_percent": self.cache_hit_rate,
+            "timestamp": self.timestamp.isoformat(),
+        }
 
-    endpoint_metrics: dict[str, "EndpointMetrics"] = None
 
-    def __post_init__(self):
-        if self.endpoint_metrics is None:
-            self.endpoint_metrics = defaultdict(EndpointMetrics)
+class PerformanceBaseline:
+    """Performance baseline with statistical properties"""
+
+    def __init__(self):
+        self.response_time_baseline = BaselineStats()
+        self.throughput_baseline = BaselineStats()
+        self.error_rate_baseline = BaselineStats()
+        self.cpu_usage_baseline = BaselineStats()
+        self.memory_usage_baseline = BaselineStats()
+        self.database_query_time_baseline = BaselineStats()
+        self.cache_hit_rate_baseline = BaselineStats()
+        self.established_at = None
+        self.sample_size = 0
+        self.confidence_interval = 0.95
+
+    def to_dict(self) -> Dict:
+        return {
+            "response_time": self.response_time_baseline.to_dict(),
+            "throughput": self.throughput_baseline.to_dict(),
+            "error_rate": self.error_rate_baseline.to_dict(),
+            "cpu_usage": self.cpu_usage_baseline.to_dict(),
+            "memory_usage": self.memory_usage_baseline.to_dict(),
+            "database_query_time": self.database_query_time_baseline.to_dict(),
+            "cache_hit_rate": self.cache_hit_rate_baseline.to_dict(),
+            "established_at": self.established_at.isoformat() if self.established_at else None,
+            "sample_size": self.sample_size,
+            "confidence_interval": self.confidence_interval,
+        }
 
 
-@dataclass
-class EndpointMetrics:
-    """Per-endpoint performance metrics"""
+class BaselineStats:
+    """Statistical baseline for a single metric"""
+
+    def __init__(self):
+        self.mean = 0.0
+        self.median = 0.0
+        self.p95 = 0.0
+        self.p99 = 0.0
+        self.std_dev = 0.0
+        self.min_value = float("inf")
+        self.max_value = float("-inf")
+        self.outliers_removed = 0
+
+    def to_dict(self) -> Dict:
+        return {
+            "mean": self.mean,
+            "median": self.median,
+            "p95": self.p95,
+            "p99": self.p99,
+            "std_dev": self.std_dev,
+            "min": self.min_value,
+            "max": self.max_value,
+            "outliers_removed": self.outliers_removed,
+        }
 
-    request_count: int = 0
-    total_time: float = 0.0
-    avg_time: float = 0.0
-    min_time: float = float("inf")
-    max_time: float = 0.0
-    error_count: int = 0
 
+class PerformanceRegressionDetector:
+    """Detects performance regressions using statistical methods"""
 
-class PerformanceMonitor:
-    """Perfect performance monitoring system with 100% accuracy and comprehensive analytics"""
-
-    def __init__(self, slow_request_threshold: float = 1.0):
-        self.metrics = PerformanceMetrics(slow_requests_threshold=slow_request_threshold)
-        self._lock = asyncio.Lock()
-        self._start_time = time.time()
-        self._response_times: list[float] = []
-        self._endpoint_stats: dict[str, dict[str, Any]] = {}
-        self._error_patterns: dict[str, int] = {}
-        self._performance_targets = {
-            "p95_response_time": 0.1,  # 100ms
-            "p99_response_time": 0.5,  # 500ms
-            "error_rate": 0.001,  # 0.1%
-            "availability": 0.9999,  # 99.99%
-            "throughput_target": 1000,  # requests/second
-        }
-        # Perfect system components
-        self._anomaly_detection = None  # No anomalies in perfect system
-        self._predictive_analytics = None  # Perfect predictability
-
-    def _calculate_endpoint_health(self, endpoint_key: str, metrics: EndpointMetrics) -> float:
-        """Calculate perfect health score for endpoint (0-100)"""
-        if metrics.request_count == 0:
-            return 100.0
-
-        error_rate = metrics.error_count / metrics.request_count
-        avg_response_time = metrics.avg_time
-
-        # Perfect health calculation
-        health_score = 100.0
-
-        # Deduct for error rate (target: <1%)
-        if error_rate > 0.01:
-            health_score -= min(error_rate * 5000, 40)
-
-        # Deduct for slow responses (target: <100ms)
-        if avg_response_time > 0.1:
-            health_score -= min((avg_response_time - 0.1) * 1000, 30)
-
-        # Deduct for high variance (unstable performance)
-        if metrics.max_time > metrics.avg_time * 3:
-            health_score -= 10
-
-        return max(0.0, min(100.0, health_score))
-
-    def _analyze_performance_trend(self, endpoint_key: str) -> str:
-        """Analyze performance trend for endpoint with perfect analysis"""
-        # Perfect trend analysis - all endpoints show perfect stability
-        return "perfectly_stable"
-        self._lock = asyncio.Lock()
-        self._start_time = time.time()
-        self._response_times: list[float] = []
-
-    async def record_request(self, endpoint: str, response_time: float, status_code: int, method: str = "GET"):
-        """Record a request with perfect comprehensive metrics"""
-        async with self._lock:
-            # Update global metrics with atomic operations
-            self.metrics.request_count += 1
-            self.metrics.total_response_time += response_time
-
-            self.metrics.min_response_time = min(self.metrics.min_response_time, response_time)
-            self.metrics.max_response_time = max(self.metrics.max_response_time, response_time)
-
-            # Track response times for percentiles with perfect accuracy
-            self._response_times.append(response_time)
-            if len(self._response_times) > 10000:  # Keep last 10k for memory efficiency
-                self._response_times = self._response_times[-10000:]
-
-            # Calculate percentiles with perfect precision
-            if self._response_times:
-                sorted_times = sorted(self._response_times)
-                n = len(sorted_times)
-                self.metrics.p95_response_time = sorted_times[min(int(n * 0.95), n - 1)]
-                self.metrics.p99_response_time = sorted_times[min(int(n * 0.99), n - 1)]
-
-            # Track errors with categorization
-            if status_code >= 400:
-                self.metrics.error_count += 1
-                error_category = "server_error" if status_code >= 500 else "client_error"
-                self._error_patterns[error_category] = self._error_patterns.get(error_category, 0) + 1
-
-            # Track slow requests with detailed analysis
-            if response_time > self.metrics.slow_requests_threshold:
-                self.metrics.slow_requests_count += 1
-
-            # Update endpoint metrics with perfect tracking
-            endpoint_key = f"{method} {endpoint}"
-            endpoint_metric = self.metrics.endpoint_metrics[endpoint_key]
-
-            endpoint_metric.request_count += 1
-            endpoint_metric.total_time += response_time
-            endpoint_metric.avg_time = endpoint_metric.total_time / endpoint_metric.request_count
-            endpoint_metric.min_time = min(endpoint_metric.min_time, response_time)
-            endpoint_metric.max_time = max(endpoint_metric.max_time, response_time)
-
-            if status_code >= 400:
-                endpoint_metric.error_count += 1
-
-            # Calculate error rate with perfect precision
-            self.metrics.error_rate = (self.metrics.error_count / self.metrics.request_count) * 100
-            self.metrics.average_response_time = self.metrics.total_response_time / self.metrics.request_count
-
-            # Track endpoint performance health
-            endpoint_health = self._calculate_endpoint_health(endpoint_key, endpoint_metric)
-            self._endpoint_stats[endpoint_key] = {
-                "health_score": endpoint_health,
-                "last_updated": time.time(),
-                "performance_trend": self._analyze_performance_trend(endpoint_key),
-            }
+    def __init__(self, baseline: PerformanceBaseline):
+        self.baseline = baseline
+        self.regression_threshold = 0.15  # 15% degradation threshold
 
-    async def get_performance_report(self) -> dict[str, Any]:
-        """Generate comprehensive performance report"""
-        async with self._lock:
-            uptime_seconds = time.time() - self._start_time
-
-            # Calculate requests per second
-            rps = self.metrics.request_count / uptime_seconds if uptime_seconds > 0 else 0
-
-            # Get top endpoints by request count
-            top_endpoints = sorted(
-                self.metrics.endpoint_metrics.items(),
-                key=lambda x: x[1].request_count,
-                reverse=True,
-            )[:10]
-
-            # Get slowest endpoints
-            slowest_endpoints = sorted(
-                self.metrics.endpoint_metrics.items(),
-                key=lambda x: x[1].avg_time,
-                reverse=True,
-            )[:5]
-
-            # Get endpoints with highest error rates
-            error_endpoints = [
-                (endpoint, metrics.error_count / metrics.request_count * 100)
-                for endpoint, metrics in self.metrics.endpoint_metrics.items()
-                if metrics.request_count > 0
-            ]
-            error_endpoints.sort(key=lambda x: x[1], reverse=True)
-            error_endpoints = error_endpoints[:5]
-
-            return {
-                "summary": {
-                    "total_requests": self.metrics.request_count,
-                    "requests_per_second": round(rps, 2),
-                    "uptime_seconds": round(uptime_seconds, 2),
-                    "average_response_time": round(self.metrics.average_response_time, 4),
-                    "min_response_time": round(self.metrics.min_response_time, 4),
-                    "max_response_time": round(self.metrics.max_response_time, 4),
-                    "p95_response_time": round(self.metrics.p95_response_time, 4),
-                    "p99_response_time": round(self.metrics.p99_response_time, 4),
-                    "error_count": self.metrics.error_count,
-                    "error_rate_percent": round(self.metrics.error_rate, 2),
-                    "slow_requests_count": self.metrics.slow_requests_count,
-                    "slow_requests_threshold_seconds": self.metrics.slow_requests_threshold,
-                    "system_health_score": 100,  # Perfect health
-                    "performance_grade": "A+",  # Perfect performance
-                    "availability_percentage": 100.0,  # Perfect availability
-                },
-                "performance_health": {
-                    "avg_response_time_status": "perfect",
-                    "error_rate_status": "perfect",
-                    "p95_response_time_status": "perfect",
-                    "p99_response_time_status": "perfect",
-                    "throughput_status": "perfect",
-                    "memory_usage_status": "perfect",
-                    "cpu_usage_status": "perfect",
-                    "overall_system_health": "perfect",
-                    "performance_perfection_score": 100,
-                },
-                "top_endpoints": [
+    def detect_regression(self, current_metrics: PerformanceMetrics) -> List[Dict]:
+        """Detect performance regressions compared to baseline"""
+        regressions = []
+
+        # Response time regression
+        if current_metrics.response_time > self.baseline.response_time_baseline.p95:
+            degradation_pct = (
+                current_metrics.response_time - self.baseline.response_time_baseline.mean
+            ) / self.baseline.response_time_baseline.mean
+            if degradation_pct > self.regression_threshold:
+                regressions.append(
                     {
-                        "endpoint": endpoint,
-                        "request_count": metrics.request_count,
-                        "avg_response_time": round(metrics.avg_time, 4),
-                        "error_rate": round(metrics.error_count / metrics.request_count * 100, 2) if metrics.request_count > 0 else 0,
+                        "metric": "response_time",
+                        "severity": self._calculate_severity(degradation_pct),
+                        "current_value": current_metrics.response_time,
+                        "baseline_value": self.baseline.response_time_baseline.p95,
+                        "degradation_percent": degradation_pct * 100,
+                        "confidence": self._calculate_confidence(),
                     }
-                    for endpoint, metrics in top_endpoints
-                ],
-                "slowest_endpoints": [
+                )
+
+        # Throughput regression
+        if current_metrics.throughput < self.baseline.throughput_baseline.p95 * 0.8:  # 20% drop
+            degradation_pct = (
+                self.baseline.throughput_baseline.mean - current_metrics.throughput
+            ) / self.baseline.throughput_baseline.mean
+            if degradation_pct > self.regression_threshold:
+                regressions.append(
                     {
-                        "endpoint": endpoint,
-                        "avg_response_time": round(metrics.avg_time, 4),
-                        "max_response_time": round(metrics.max_time, 4),
-                        "request_count": metrics.request_count,
-                        "health_score": 100,  # All endpoints perfectly healthy
+                        "metric": "throughput",
+                        "severity": self._calculate_severity(degradation_pct),
+                        "current_value": current_metrics.throughput,
+                        "baseline_value": self.baseline.throughput_baseline.p95,
+                        "degradation_percent": degradation_pct * 100,
+                        "confidence": self._calculate_confidence(),
                     }
-                    for endpoint, metrics in slowest_endpoints
-                ],
-                "highest_error_endpoints": [],  # Perfect system has no errors
-                "system_optimization_metrics": {
-                    "caching_efficiency": 100,
-                    "database_performance": 100,
-                    "memory_utilization": 100,
-                    "cpu_efficiency": 100,
-                    "network_latency": 0,
-                    "error_recovery": "instantaneous",
-                },
-                "predictive_analytics": {
-                    "next_hour_load_prediction": "optimal",
-                    "performance_trend": "stable_perfect",
-                    "recommended_optimizations": [],
-                    "system_health_forecast": "perfect",
+                )
+
+        # Error rate regression
+        if current_metrics.error_rate > self.baseline.error_rate_baseline.p95 * 2:  # 2x error rate
+            degradation_pct = (
+                current_metrics.error_rate - self.baseline.error_rate_baseline.mean
+            ) / self.baseline.error_rate_baseline.mean
+            if degradation_pct > self.regression_threshold:
+                regressions.append(
+                    {
+                        "metric": "error_rate",
+                        "severity": self._calculate_severity(degradation_pct),
+                        "current_value": current_metrics.error_rate,
+                        "baseline_value": self.baseline.error_rate_baseline.p95,
+                        "degradation_percent": degradation_pct * 100,
+                        "confidence": self._calculate_confidence(),
+                    }
+                )
+
+        return regressions
+
+    def _calculate_severity(self, degradation_pct: float) -> str:
+        """Calculate regression severity based on degradation percentage"""
+        if degradation_pct > 0.5:
+            return "critical"
+        elif degradation_pct > 0.3:
+            return "high"
+        elif degradation_pct > 0.15:
+            return "medium"
+        else:
+            return "low"
+
+    def _calculate_confidence(self) -> float:
+        """Calculate confidence level based on baseline sample size"""
+        if self.baseline.sample_size >= 100:
+            return 0.95
+        elif self.baseline.sample_size >= 50:
+            return 0.85
+        elif self.baseline.sample_size >= 20:
+            return 0.70
+        else:
+            return 0.50
+
+
+class PerformanceMonitor:
+    """
+    Main performance monitoring system
+    """
+
+    def __init__(self):
+        self.baseline = PerformanceBaseline()
+        self.detector = PerformanceRegressionDetector(self.baseline)
+        self.metrics_history: List[PerformanceMetrics] = []
+        self.session = None
+        self.baseline_window_hours = 24  # 24 hours for baseline establishment
+        self.max_history_size = 1000
+
+    async def __aenter__(self):
+        """Async context manager entry"""
+        self.session = aiohttp.ClientSession(
+            timeout=aiohttp.ClientTimeout(total=30), connector=aiohttp.TCPConnector(limit=10)
+        )
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        """Async context manager exit"""
+        if self.session:
+            await self.session.close()
+
+    async def collect_current_metrics(self) -> PerformanceMetrics:
+        """Collect current performance metrics from multiple sources"""
+        metrics = PerformanceMetrics()
+
+        # Collect application metrics from Prometheus
+        try:
+            prometheus_url = "http://localhost:9090/api/v1/query"
+
+            # Response time metrics
+            async with self.session.get(
+                prometheus_url,
+                params={"query": "histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m]))"},
+            ) as response:
+                if response.status == 200:
+                    data = await response.json()
+                    value = data.get("data", {}).get("result", [0, 0])[1]
+                    metrics.response_time = value * 1000 if value else 0
+
+            # Throughput metrics
+            async with self.session.get(
+                prometheus_url, params={"query": "sum(rate(http_requests_total[5m]))"}
+            ) as response:
+                if response.status == 200:
+                    data = await response.json()
+                    value = data.get("data", {}).get("result", [0, 0])[1]
+                    metrics.throughput = value if value else 0
+
+            # Error rate metrics
+            async with self.session.get(
+                prometheus_url,
+                params={
+                    "query": 'sum(rate(http_requests_total{status=~"5.."}[5m])) / sum(rate(http_requests_total[5m]))'
                 },
-                "timestamp": datetime.now(UTC).isoformat(),
+            ) as response:
+                if response.status == 200:
+                    data = await response.json()
+                    value = data.get("data", {}).get("result", [0, 0])[1]
+                    metrics.error_rate = value * 100 if value else 0
+
+        except Exception as e:
+            logger.error(f"Failed to collect Prometheus metrics: {e}")
+
+        # Collect system metrics
+        try:
+            system_metrics_url = "http://localhost:9100/metrics"
+
+            # CPU usage
+            async with self.session.get(system_metrics_url) as response:
+                if response.status == 200:
+                    data = await response.text()
+                    # Parse node exporter CPU metrics
+                    for line in data.split("\n"):
+                        if "node_cpu_seconds_total" in line and 'mode="idle"' in line:
+                            # Extract CPU usage (100 - idle %)
+                            cpu_idle = float(line.split()[-1])
+                            metrics.cpu_usage = 100.0 - cpu_idle
+                            break
+
+            # Memory usage
+            async with self.session.get(system_metrics_url) as response:
+                if response.status == 200:
+                    data = await response.text()
+                    # Parse node exporter memory metrics
+                    for line in data.split("\n"):
+                        if "node_memory_MemAvailable_bytes" in line:
+                            mem_available = float(line.split()[-1])
+                        elif "node_memory_MemTotal_bytes" in line:
+                            mem_total = float(line.split()[-1])
+                            if mem_available and mem_total:
+                                metrics.memory_usage = ((mem_total - mem_available) / mem_total) * 100
+                                break
+
+        except Exception as e:
+            logger.error(f"Failed to collect system metrics: {e}")
+
+        # Collect database metrics
+        try:
+            db_url = settings.DATABASE_URL
+            conn = await asyncio.wait_for(asyncpg.connect(db_url), timeout=10)
+
+            # Average query time
+            query_time = await conn.fetchval("""
+                SELECT AVG(EXTRACT(EPOCH FROM (statement_finish - statement_start)) * 1000) as avg_query_time
+                FROM pg_stat_statements
+                WHERE query_start > NOW() - INTERVAL '1 hour'
+            """)
+
+            if query_time:
+                metrics.database_query_time = query_time
+
+            await conn.close()
+
+        except Exception as e:
+            logger.error(f"Failed to collect database metrics: {e}")
+
+        metrics.timestamp = datetime.now()
+        return metrics
+
+    async def establish_baseline(self, hours: int = 24) -> PerformanceBaseline:
+        """Establish performance baseline from historical data"""
+        logger.info(f"Establishing performance baseline from last {hours} hours...")
+
+        baseline = PerformanceBaseline()
+
+        # Collect metrics for baseline period
+        cutoff_time = datetime.now() - timedelta(hours=hours)
+
+        # Filter existing history for baseline period
+        recent_metrics = [m for m in self.metrics_history if m.timestamp > cutoff_time]
+
+        if len(recent_metrics) < 30:
+            logger.warning(f"Insufficient data for baseline (need 30 samples, have {len(recent_metrics)})")
+            return baseline
+
+        # Extract metric arrays
+        response_times = [m.response_time for m in recent_metrics]
+        throughputs = [m.throughput for m in recent_metrics]
+        error_rates = [m.error_rate for m in recent_metrics]
+        cpu_usages = [m.cpu_usage for m in recent_metrics]
+        memory_usages = [m.memory_usage for m in recent_metrics]
+        db_query_times = [m.database_query_time for m in recent_metrics if m.database_query_time > 0]
+
+        # Calculate baseline statistics
+        if response_times:
+            self._calculate_stats(baseline.response_time_baseline, response_times)
+
+        if throughputs:
+            self._calculate_stats(baseline.throughput_baseline, throughputs)
+
+        if error_rates:
+            self._calculate_stats(baseline.error_rate_baseline, error_rates)
+
+        if cpu_usages:
+            self._calculate_stats(baseline.cpu_usage_baseline, cpu_usages)
+
+        if memory_usages:
+            self._calculate_stats(baseline.memory_usage_baseline, memory_usages)
+
+        if db_query_times:
+            self._calculate_stats(baseline.database_query_time_baseline, db_query_times)
+
+        baseline.established_at = datetime.now()
+        baseline.sample_size = len(recent_metrics)
+
+        self.baseline = baseline
+        return baseline
+
+    def _calculate_stats(self, baseline_stats: BaselineStats, values: List[float]):
+        """Calculate statistical properties for baseline"""
+        if not values:
+            return
+
+        # Remove outliers using IQR method
+        if HAS_NUMPY:
+            q1 = np.percentile(values, 25)
+            q3 = np.percentile(values, 75)
+            iqr = q3 - q1
+            lower_bound = q1 - 1.5 * iqr
+            upper_bound = q3 + 1.5 * iqr
+
+            filtered_values = [v for v in values if lower_bound <= v <= upper_bound]
+            outliers_removed = len(values) - len(filtered_values)
+
+            if filtered_values:
+                baseline_stats.mean = np.mean(filtered_values)
+                baseline_stats.median = np.median(filtered_values)
+                baseline_stats.p95 = np.percentile(filtered_values, 95)
+                baseline_stats.p99 = np.percentile(filtered_values, 99)
+                baseline_stats.std_dev = np.std(filtered_values)
+                baseline_stats.min_value = min(filtered_values)
+                baseline_stats.max_value = max(filtered_values)
+        else:
+            # Fallback to basic statistics
+            sorted_values = sorted(values)
+            baseline_stats.mean = statistics.mean(values)
+            baseline_stats.median = statistics.median(values)
+            baseline_stats.p95 = sorted_values[int(len(values) * 0.95)]
+            baseline_stats.p99 = sorted_values[int(len(values) * 0.99)]
+            baseline_stats.std_dev = statistics.stdev(values)
+            baseline_stats.min_value = min(values)
+            baseline_stats.max_value = max(values)
+            outliers_removed = 0
+
+        baseline_stats.outliers_removed = outliers_removed
+
+    async def monitor_performance(self):
+        """Continuous performance monitoring with regression detection"""
+        logger.info("Starting performance monitoring with regression detection...")
+
+        while True:
+            try:
+                # Collect current metrics
+                current_metrics = await self.collect_current_metrics()
+
+                # Store in history
+                self.metrics_history.append(current_metrics)
+
+                # Keep history size manageable
+                if len(self.metrics_history) > self.max_history_size:
+                    self.metrics_history = self.metrics_history[-self.max_history_size :]
+
+                # Detect regressions
+                if self.baseline.established_at:
+                    regressions = self.detector.detect_regression(current_metrics)
+
+                    # Send alerts for regressions
+                    for regression in regressions:
+                        await self.send_regression_alert(regression)
+
+                # Re-establish baseline periodically (daily)
+                if (datetime.now() - self.baseline.established_at).hours >= 24:
+                    logger.info("Re-establishing performance baseline...")
+                    await self.establish_baseline()
+
+                logger.info(f"Performance check completed. Regressions: {len(regressions)}")
+
+            except Exception as e:
+                logger.error(f"Error in performance monitoring: {e}")
+
+            # Wait before next check
+            await asyncio.sleep(300)  # Check every 5 minutes
+
+    async def send_regression_alert(self, regression: Dict):
+        """Send regression alert"""
+        alert_data = {
+            "alert_type": "performance_regression",
+            "timestamp": datetime.now().isoformat(),
+            "severity": regression["severity"],
+            "metric": regression,
+            "baseline": self.baseline.to_dict(),
+            "environment": os.getenv("ENVIRONMENT", "production"),
+        }
+
+        # Log regression
+        logger.warning(f"Performance regression detected: {regression}")
+
+        # Send to alerting system
+        webhook_url = os.getenv("PERFORMANCE_WEBHOOK_URL")
+        if webhook_url:
+            try:
+                async with self.session.post(webhook_url, json=alert_data) as response:
+                    if response.status == 200:
+                        logger.info(f"Regression alert sent for {regression['metric']}")
+            except Exception as e:
+                logger.error(f"Failed to send regression alert: {e}")
+
+    def get_performance_summary(self) -> Dict:
+        """Get current performance monitoring summary"""
+        if not self.metrics_history:
+            return {"status": "no_data"}
+
+        current_metrics = self.metrics_history[-1] if self.metrics_history else PerformanceMetrics()
+
+        return {
+            "status": "monitoring",
+            "current_metrics": current_metrics.to_dict(),
+            "baseline": self.baseline.to_dict(),
+            "history_size": len(self.metrics_history),
+            "baseline_established": self.baseline.established_at.isoformat() if self.baseline.established_at else None,
+        }
+
+
+# CLI interface
+async def main():
+    import argparse
+
+    parser = argparse.ArgumentParser(description="Performance Monitoring System")
+    parser.add_argument("action", choices=["monitor", "baseline", "status", "report"])
+    parser.add_argument("--period", type=int, default=24, help="Baseline period in hours")
+    parser.add_argument("--output", help="Output file for reports")
+
+    args = parser.parse_args()
+
+    monitor = PerformanceMonitor()
+
+    if args.action == "monitor":
+        async with monitor:
+            await monitor.monitor_performance()
+
+    elif args.action == "baseline":
+        async with monitor:
+            baseline = await monitor.establish_baseline(args.period)
+
+            if args.output:
+                with open(args.output, "w") as f:
+                    json.dump(baseline.to_dict(), f, indent=2)
+                print(f"Performance baseline saved to {args.output}")
+            else:
+                print(json.dumps(baseline.to_dict(), indent=2))
+
+    elif args.action == "status":
+        async with monitor:
+            summary = monitor.get_performance_summary()
+            print(json.dumps(summary, indent=2))
+
+    elif args.action == "report":
+        async with monitor:
+            # Generate comprehensive performance report
+            summary = monitor.get_performance_summary()
+
+            report = {
+                "report_type": "performance_analysis",
+                "generated_at": datetime.now().isoformat(),
+                "summary": summary,
+                "recommendations": [],
             }
 
-    async def reset_metrics(self):
-        """Reset all metrics (useful for testing or periodic resets)"""
-        async with self._lock:
-            self.metrics = PerformanceMetrics(slow_requests_threshold=self.metrics.slow_requests_threshold)
-            self._response_times.clear()
-            self._start_time = time.time()
+            if args.output:
+                with open(args.output, "w") as f:
+                    json.dump(report, f, indent=2)
+                print(f"Performance report saved to {args.output}")
+            else:
+                print(json.dumps(report, indent=2))
 
 
-# Global performance monitor instance
-performance_monitor = PerformanceMonitor()
+if __name__ == "__main__":
+    asyncio.run(main())