tests/bug_discovery/feedback_loops/roi_tracker.py

"""
ROI Tracker for Bug Discovery Automation.

This module tracks and calculates ROI metrics for automated bug discovery,
comparing manual QA costs vs automation costs, and demonstrating business value.

Example:
    tracker = ROITracker()
    tracker.record_discovery_run(
        bugs_found=42,
        unique_bugs=35,
        filed_bugs=30,
        duration_seconds=3600,
        by_method={"fuzzing": 20, "chaos": 10, "property": 8, "browser": 4},
        by_severity={"critical": 2, "high": 10, "medium": 15, "low": 15}
    )
    roi_report = tracker.generate_roi_report(weeks=4)
    print(f"ROI: {roi_report['roi_ratio']:.1f}x")
"""

import json
import sqlite3
from pathlib import Path
from typing import Dict, List, Any, Optional
from datetime import datetime, timedelta

# Add backend to path for imports
import sys
backend_dir = Path(__file__).parent.parent.parent.parent
if str(backend_dir) not in sys.path:
    sys.path.insert(0, str(backend_dir))


class ROITracker:
    """
    Track ROI metrics for bug discovery automation.

    Metrics collected:
    - Bugs found (by method, severity)
    - Time to discovery (automation vs manual QA)
    - Time to fix (from filed to closed)
    - Cost avoidance (bugs prevented from production)
    - Manual QA cost vs automation cost

    Example:
        tracker = ROITracker()
        tracker.record_discovery_run(
            bugs_found=42,
            unique_bugs=35,
            filed_bugs=30,
            duration_seconds=3600,
            by_method={"fuzzing": 20, "chaos": 10, "property": 8, "browser": 4},
            by_severity={"critical": 2, "high": 10, "medium": 15, "low": 15}
        )
        roi_report = tracker.generate_roi_report(weeks=4)
        print(f"ROI: {roi_report['roi_ratio']:.1f}x")
    """

    # Default cost assumptions (configurable via __init__)
    DEFAULT_MANUAL_QA_HOURLY_RATE = 75.0  # $/hour
    DEFAULT_DEVELOPER_HOURLY_RATE = 100.0  # $/hour
    DEFAULT_BUG_PRODUCTION_COST = 10000.0  # Average cost of production bug
    DEFAULT_MANUAL_QA_HOURS_PER_BUG = 2.0  # Hours to manually find and report a bug

    def __init__(
        self,
        db_path: str = None,
        manual_qa_hourly_rate: float = None,
        developer_hourly_rate: float = None,
        bug_production_cost: float = None,
        manual_qa_hours_per_bug: float = None
    ):
        """
        Initialize ROITracker.

        Args:
            db_path: Path to SQLite database (default: backend/tests/bug_discovery/storage/metrics.db)
            manual_qa_hourly_rate: Cost per hour for manual QA (default: $75)
            developer_hourly_rate: Cost per hour for developer time (default: $100)
            bug_production_cost: Average cost of bug reaching production (default: $10,000)
            manual_qa_hours_per_bug: Hours to manually find/report a bug (default: 2.0)
        """
        if db_path is None:
            db_path = backend_dir / "tests" / "bug_discovery" / "storage" / "metrics.db"

        self.db_path = Path(db_path)
        self.db_path.parent.mkdir(parents=True, exist_ok=True)

        # Cost assumptions
        self.manual_qa_hourly_rate = manual_qa_hourly_rate or self.DEFAULT_MANUAL_QA_HOURLY_RATE
        self.developer_hourly_rate = developer_hourly_rate or self.DEFAULT_DEVELOPER_HOURLY_RATE
        self.bug_production_cost = bug_production_cost or self.DEFAULT_BUG_PRODUCTION_COST
        self.manual_qa_hours_per_bug = manual_qa_hours_per_bug or self.DEFAULT_MANUAL_QA_HOURS_PER_BUG

        # Initialize database schema
        self._init_db()

    def _init_db(self):
        """Initialize SQLite database with metrics schema."""
        # Ensure parent directory exists
        self.db_path.parent.mkdir(parents=True, exist_ok=True)

        conn = sqlite3.connect(str(self.db_path))
        cursor = conn.cursor()

        # Discovery runs table
        cursor.execute("""
            CREATE TABLE IF NOT EXISTS discovery_runs (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                timestamp TEXT NOT NULL,
                bugs_found INTEGER NOT NULL,
                unique_bugs INTEGER NOT NULL,
                filed_bugs INTEGER NOT NULL,
                duration_seconds REAL NOT NULL,
                by_method TEXT NOT NULL,
                by_severity TEXT NOT NULL,
                automation_cost REAL NOT NULL
            )
        """)

        # Bug fixes table
        cursor.execute("""
            CREATE TABLE IF NOT EXISTS bug_fixes (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                bug_id TEXT NOT NULL,
                issue_number INTEGER NOT NULL,
                filed_at TEXT NOT NULL,
                fixed_at TEXT NOT NULL,
                fix_duration_hours REAL NOT NULL,
                severity TEXT NOT NULL,
                discovery_method TEXT NOT NULL
            )
        """)

        # ROI summary table (aggregated weekly)
        cursor.execute("""
            CREATE TABLE IF NOT EXISTS roi_summary (
                id INTEGER PRIMARY KEY AUTOINCREMENT,
                week_start TEXT NOT NULL UNIQUE,
                bugs_found INTEGER NOT NULL,
                bugs_fixed INTEGER NOT NULL,
                hours_saved REAL NOT NULL,
                cost_saved REAL NOT NULL,
                automation_cost REAL NOT NULL,
                roi REAL NOT NULL,
                bugs_prevented INTEGER NOT NULL,
                cost_avoidance REAL NOT NULL,
                total_savings REAL NOT NULL,
                created_at TEXT NOT NULL
            )
        """)

        conn.commit()
        conn.close()

    def record_discovery_run(
        self,
        bugs_found: int,
        unique_bugs: int,
        filed_bugs: int,
        duration_seconds: float,
        by_method: Dict[str, int],
        by_severity: Dict[str, int]
    ):
        """
        Record bug discovery run metrics.

        Args:
            bugs_found: Total bugs found (including duplicates)
            unique_bugs: Unique bugs after deduplication
            filed_bugs: Bugs filed to GitHub
            duration_seconds: Discovery run duration
            by_method: Bugs by discovery method (e.g., {"fuzzing": 10, "chaos": 5})
            by_severity: Bugs by severity (e.g., {"critical": 2, "high": 8})
        """
        conn = sqlite3.connect(str(self.db_path))
        cursor = conn.cursor()

        # Calculate automation cost (developer time)
        automation_cost = (duration_seconds / 3600) * self.developer_hourly_rate

        cursor.execute("""
            INSERT INTO discovery_runs (
                timestamp, bugs_found, unique_bugs, filed_bugs,
                duration_seconds, by_method, by_severity, automation_cost
            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
        """, (
            datetime.utcnow().isoformat(),
            bugs_found,
            unique_bugs,
            filed_bugs,
            duration_seconds,
            json.dumps(by_method),
            json.dumps(by_severity),
            automation_cost
        ))

        conn.commit()
        conn.close()

    def record_fixes(
        self,
        bug_ids: List[str],
        issue_numbers: List[int],
        filed_dates: List[str],
        fix_duration_hours: float,
        severity: str = "medium",
        discovery_method: str = "unknown"
    ):
        """
        Record bug fix metrics.

        Args:
            bug_ids: List of bug IDs
            issue_numbers: List of GitHub issue numbers
            filed_dates: List of filed dates (ISO format)
            fix_duration_hours: Average time to fix (hours)
            severity: Bug severity (default: "medium")
            discovery_method: How bug was discovered (default: "unknown")
        """
        conn = sqlite3.connect(str(self.db_path))
        cursor = conn.cursor()

        for bug_id, issue_number, filed_date in zip(bug_ids, issue_numbers, filed_dates):
            cursor.execute("""
                INSERT INTO bug_fixes (
                    bug_id, issue_number, filed_at, fixed_at, fix_duration_hours, severity, discovery_method
                ) VALUES (?, ?, ?, ?, ?, ?, ?)
            """, (
                bug_id,
                issue_number,
                filed_date,
                datetime.utcnow().isoformat(),
                fix_duration_hours,
                severity,
                discovery_method
            ))

        conn.commit()
        conn.close()

    def generate_roi_report(
        self,
        weeks: int = 4,
        include_breakdown: bool = True
    ) -> Dict[str, Any]:
        """
        Generate ROI report for last N weeks.

        Args:
            weeks: Number of weeks to include in report (default: 4)
            include_breakdown: Include detailed breakdown by method/severity

        Returns:
            Dict with ROI metrics (bugs_found, bugs_fixed, hours_saved, cost_saved, roi_ratio)
        """
        conn = sqlite3.connect(str(self.db_path))
        cursor = conn.cursor()

        # Calculate week start date
        week_start = (datetime.utcnow() - timedelta(weeks=weeks)).isoformat()

        # Aggregate discovery metrics
        cursor.execute("""
            SELECT
                SUM(bugs_found) as total_bugs_found,
                SUM(unique_bugs) as total_unique_bugs,
                SUM(filed_bugs) as total_filed_bugs,
                SUM(duration_seconds) as total_duration_seconds,
                SUM(automation_cost) as total_automation_cost
            FROM discovery_runs
            WHERE timestamp >= ?
        """, (week_start,))

        discovery_row = cursor.fetchone()

        # Aggregate fix metrics
        cursor.execute("""
            SELECT
                COUNT(*) as total_fixed_bugs,
                AVG(fix_duration_hours) as avg_fix_duration_hours
            FROM bug_fixes
            WHERE fixed_at >= ?
        """, (week_start,))

        fix_row = cursor.fetchone()

        conn.close()

        # Extract values
        total_bugs_found = discovery_row[0] or 0
        total_unique_bugs = discovery_row[1] or 0
        total_filed_bugs = discovery_row[2] or 0
        total_duration_seconds = discovery_row[3] or 0
        total_automation_cost = discovery_row[4] or 0
        total_fixed_bugs = fix_row[0] or 0
        avg_fix_duration_hours = fix_row[1] or 0

        # Calculate ROI metrics
        manual_qa_hours = total_bugs_found * self.manual_qa_hours_per_bug
        automation_hours = total_duration_seconds / 3600
        hours_saved = manual_qa_hours - automation_hours

        # Cost calculations
        manual_qa_cost = manual_qa_hours * self.manual_qa_hourly_rate
        cost_saved = manual_qa_cost - total_automation_cost

        # Bugs prevented from production (10% assumption)
        bugs_prevented = int(total_bugs_found * 0.1)
        cost_avoidance = bugs_prevented * self.bug_production_cost

        # Total savings
        total_savings = cost_saved + cost_avoidance

        # ROI ratio (savings / cost)
        roi_ratio = total_savings / total_automation_cost if total_automation_cost > 0 else 0

        report = {
            "period_weeks": weeks,
            "period_start": week_start,
            "period_end": datetime.utcnow().isoformat(),
            "bugs_found": total_bugs_found,
            "unique_bugs": total_unique_bugs,
            "filed_bugs": total_filed_bugs,
            "bugs_fixed": total_fixed_bugs,
            "manual_qa_hours": manual_qa_hours,
            "automation_hours": automation_hours,
            "hours_saved": hours_saved,
            "manual_qa_cost": manual_qa_cost,
            "automation_cost": total_automation_cost,
            "cost_saved": cost_saved,
            "bugs_prevented": bugs_prevented,
            "cost_avoidance": cost_avoidance,
            "total_savings": total_savings,
            "roi_ratio": roi_ratio,
            "avg_fix_duration_hours": avg_fix_duration_hours,
            "cost_assumptions": {
                "manual_qa_hourly_rate": self.manual_qa_hourly_rate,
                "developer_hourly_rate": self.developer_hourly_rate,
                "bug_production_cost": self.bug_production_cost,
                "manual_qa_hours_per_bug": self.manual_qa_hours_per_bug
            }
        }

        # Add breakdown if requested
        if include_breakdown:
            report["breakdown"] = self._get_breakdown(week_start)

        return report

    def _get_breakdown(self, since: str) -> Dict[str, Any]:
        """Get detailed breakdown by method and severity."""
        conn = sqlite3.connect(str(self.db_path))
        cursor = conn.cursor()

        # By method breakdown
        cursor.execute("""
            SELECT by_method FROM discovery_runs WHERE timestamp >= ?
        """, (since,))

        method_counts = {}
        for (by_method_json,) in cursor.fetchall():
            by_method = json.loads(by_method_json)
            for method, count in by_method.items():
                method_counts[method] = method_counts.get(method, 0) + count

        # By severity breakdown
        cursor.execute("""
            SELECT by_severity FROM discovery_runs WHERE timestamp >= ?
        """, (since,))

        severity_counts = {}
        for (by_severity_json,) in cursor.fetchall():
            by_severity = json.loads(by_severity_json)
            for severity, count in by_severity.items():
                severity_counts[severity] = severity_counts.get(severity, 0) + count

        conn.close()

        return {
            "by_method": method_counts,
            "by_severity": severity_counts
        }

    def get_weekly_trends(
        self,
        weeks: int = 12
    ) -> List[Dict[str, Any]]:
        """
        Get weekly trend data for charts.

        Args:
            weeks: Number of weeks to include

        Returns:
            List of weekly data points
        """
        conn = sqlite3.connect(str(self.db_path))
        cursor = conn.cursor()

        trends = []
        for i in range(weeks):
            week_start = (datetime.utcnow() - timedelta(weeks=i+1)).isoformat()
            week_end = (datetime.utcnow() - timedelta(weeks=i)).isoformat()

            cursor.execute("""
                SELECT
                    SUM(bugs_found) as bugs_found,
                    SUM(unique_bugs) as unique_bugs,
                    SUM(filed_bugs) as filed_bugs,
                    SUM(automation_cost) as automation_cost
                FROM discovery_runs
                WHERE timestamp >= ? AND timestamp < ?
            """, (week_start, week_end))

            row = cursor.fetchone()
            trends.append({
                "week_start": week_start,
                "week_end": week_end,
                "bugs_found": row[0] or 0,
                "unique_bugs": row[1] or 0,
                "filed_bugs": row[2] or 0,
                "automation_cost": row[3] or 0
            })

        conn.close()
        return list(reversed(trends))

    def save_weekly_summary(self, report: Dict[str, Any]):
        """
        Save weekly ROI summary to database.

        Args:
            report: ROI report from generate_roi_report()
        """
        conn = sqlite3.connect(str(self.db_path))
        cursor = conn.cursor()

        # Calculate week start (Monday of current week)
        today = datetime.utcnow().date()
        week_start = (today - timedelta(days=today.weekday())).isoformat()

        try:
            cursor.execute("""
                INSERT INTO roi_summary (
                    week_start, bugs_found, bugs_fixed, hours_saved, cost_saved,
                    automation_cost, roi, bugs_prevented, cost_avoidance, total_savings, created_at
                ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
            """, (
                week_start,
                report["bugs_found"],
                report["bugs_fixed"],
                report["hours_saved"],
                report["cost_saved"],
                report["automation_cost"],
                report["roi_ratio"],
                report["bugs_prevented"],
                report["cost_avoidance"],
                report["total_savings"],
                datetime.utcnow().isoformat()
            ))
            conn.commit()
        except sqlite3.IntegrityError:
            # Week already exists, update instead
            cursor.execute("""
                UPDATE roi_summary SET
                    bugs_found = ?, bugs_fixed = ?, hours_saved = ?, cost_saved = ?,
                    automation_cost = ?, roi = ?, bugs_prevented = ?, cost_avoidance = ?, total_savings = ?
                WHERE week_start = ?
            """, (
                report["bugs_found"],
                report["bugs_fixed"],
                report["hours_saved"],
                report["cost_saved"],
                report["automation_cost"],
                report["roi_ratio"],
                report["bugs_prevented"],
                report["cost_avoidance"],
                report["total_savings"],
                week_start
            ))
            conn.commit()

        conn.close()