fix AI model

check_ai_only.py

@@ -0,0 +1,49 @@
+"""Verify AI-only mode (run_anomaly_detection=False, use_model=True):
+- detect_anomalies should run only the iforest, no rule detectors.
+- Every row in the response should carry has_anomaly=YES with a method
+  starting "AI:".
+"""
+import sys
+from pathlib import Path
+sys.path.insert(0, str(Path(__file__).parent))
+
+from telecom_anomaly.core.detector import TelecomAnomalyDetector
+from telecom_anomaly.core.models import TelecomConfig
+
+CSV = (
+    r"C:\my_project\work\GH_Police\main project\backend"
+    r"\telecom_anomaly_detection\data\strictly_cleaned_data"
+    r"\591296805 calls (1)_strictly_cleaned.csv"
+)
+
+cfg = TelecomConfig()
+det = TelecomAnomalyDetector(cfg)
+det.load_data([CSV])
+
+# AI-only mode
+det.detect_anomalies(use_model=True, run_rules=False)
+stats = det.get_statistics()
+
+print("=== AI-only mode (run_rules=False, use_model=True) ===")
+print(f"Total anomalies: {stats.total_anomalies_detected}")
+print(f"Rows with anomalies: {stats.rows_with_anomalies}")
+print(f"anomaly_types (should be AI: only): {stats.anomaly_types}")
+
+# Confirm no rule detector fired
+rule_method_seen = any(
+    not m.startswith('AI:')
+    for methods in det.row_anomaly_map.values()
+    for m in methods
+)
+print(f"Any non-AI method fired? {rule_method_seen}")
+
+# Show that 5 sample rows would land with has_anomaly=YES in the response
+print("\nSample of 5 flagged rows (these would have has_anomaly=YES):")
+shown = 0
+for orig_idx, methods in det.row_anomaly_map.items():
+    if shown >= 5:
+        break
+    row = det.original_data[orig_idx]
+    has_anomaly = "YES" if methods else "NO"
+    print(f"  row {orig_idx}: has_anomaly={has_anomaly} methods={methods}")
+    shown += 1

check_model_summary.py

@@ -0,0 +1,44 @@
+"""Print the full model_summary block the API will emit at the end of
+every detection that runs the iforest. Verifies every key the planning
+doc promises is populated and well-formed.
+"""
+import json
+import sys
+from pathlib import Path
+sys.path.insert(0, str(Path(__file__).parent))
+
+from telecom_anomaly.core.detector import TelecomAnomalyDetector
+from telecom_anomaly.core.models import TelecomConfig
+
+CSV = (
+    r"C:\my_project\work\GH_Police\main project\backend"
+    r"\telecom_anomaly_detection\data\strictly_cleaned_data"
+    r"\591296805 calls (1)_strictly_cleaned.csv"
+)
+
+cfg = TelecomConfig()
+det = TelecomAnomalyDetector(cfg)
+det.load_data([CSV])
+
+# Cover all three modes.
+for label, kwargs in [
+    ("Rules-only",  {"use_model": False, "run_rules": True}),
+    ("AI-only",     {"use_model": True,  "run_rules": False}),
+    ("Both",        {"use_model": True,  "run_rules": True}),
+]:
+    # Reset tracking between modes.
+    det.anomalies = []
+    det.rows_with_anomalies = set()
+    det.row_anomaly_map.clear()
+    det.row_anomaly_reasons.clear()
+    det.row_anomaly_attribution.clear()
+    det.subscriber_anomalies.clear()
+
+    det.detect_anomalies(**kwargs)
+    stats = det.get_statistics()
+    print(f"\n=== {label} ({kwargs}) ===")
+    print(f"total_anomalies_detected: {stats.total_anomalies_detected}")
+    print(f"anomaly_types           : {stats.anomaly_types}")
+    print(f"model_feature_drivers   : {stats.model_feature_drivers}")
+    print("model_summary:")
+    print(json.dumps(stats.model_summary, indent=2, default=str))

telecom_anomaly/api.py

@@ -209,8 +209,18 @@ def _run_analysis(
         logger.info(f"Processing CSV: {original_filename}")
         detector.load_data([tmp_path], filter_movements=filter_movements)
 
-    if run_anomaly_detection:
-        detector.detect_anomalies(use_model=use_model)
+    # Detection runs when EITHER the rule-based pipeline ("Anomaly
+    # Detection") OR the model pipeline ("Use AI Model") was requested.
+    # `run_rules=run_anomaly_detection` makes the rule detectors honour
+    # the rule toggle independently, so the four combinations work as
+    # expected: rules-only, model-only, both, or neither (clustering /
+    # preview only).
+    any_detection = run_anomaly_detection or use_model
+    if any_detection:
+        detector.detect_anomalies(
+            use_model=use_model,
+            run_rules=run_anomaly_detection,
+        )
 
     stats = detector.get_statistics()
     kept_original_indices = set(detector.data_indices.values())
@@ -220,7 +230,11 @@ def _run_analysis(
         if idx not in kept_original_indices:
             continue
         methods = detector.row_anomaly_map.get(idx, [])
-        if run_anomaly_detection and not methods:
+        # When any detection ran, only flagged rows make it into the
+        # response. When no detection ran (clustering-only), every kept
+        # row goes through with has_anomaly=NO so the caller still sees
+        # the clustered dataset.
+        if any_detection and not methods:
             continue
         reasons = detector.row_anomaly_reasons.get(idx, [])
         attributions = detector.row_anomaly_attribution.get(idx, [])

telecom_anomaly/core/detector.py

@@ -198,21 +198,32 @@ class TelecomAnomalyDetector:
         
         logger.info(f"Geographic filtering complete: {original_count} -> {len(self.data)} records")
     
-    def detect_anomalies(self, use_model: bool = False) -> List[AnomalyResult]:
+    def detect_anomalies(
+        self,
+        use_model: bool = False,
+        run_rules: bool = True,
+    ) -> List[AnomalyResult]:
         """
         Run anomaly detectors over the loaded data.
 
         Args:
-            use_model: When True, also runs model-based detectors
-                (Isolation Forest). Default False keeps behaviour
-                identical to the original rule-only pipeline.
+            use_model: When True, runs the model-based detectors
+                (Isolation Forest).
+            run_rules: When True, runs the rule-based detectors. Set to
+                False to run the model alone — this is what powers the
+                Flutter "Use AI Model" toggle without "Anomaly Detection".
+                Defaults to True so existing callers (CLI, prior tests)
+                keep their rule-only behaviour.
         """
         start_time = datetime.now()
         logger.info(
-            f"Starting anomaly detection (use_model={use_model})"
+            f"Starting anomaly detection "
+            f"(run_rules={run_rules}, use_model={use_model})"
         )
 
-        detectors_to_run = list(self.rule_detectors)
+        detectors_to_run = []
+        if run_rules:
+            detectors_to_run.extend(self.rule_detectors)
         if use_model:
             detectors_to_run.extend(self.model_detectors)
 
@@ -330,6 +341,10 @@ class TelecomAnomalyDetector:
                 top = '_multivariate'
             model_feature_drivers[top] += 1
 
+        model_summary = self._build_model_summary(
+            dict(model_feature_drivers)
+        )
+
         return DetectionStatistics(
             total_rows=len(self.data),
             rows_with_anomalies=rows_with_anomalies,
@@ -338,8 +353,139 @@ class TelecomAnomalyDetector:
             unique_subscribers=len(set(a.msisdn for a in self.anomalies if a.msisdn)),
             anomaly_types=dict(anomaly_types),
             model_feature_drivers=dict(model_feature_drivers),
+            model_summary=model_summary,
         )
 
+    def _build_model_summary(
+        self,
+        feature_driver_counts: Dict[str, int],
+    ) -> Dict[str, Any]:
+        """
+        Consolidated IsolationForest report. Empty dict if no model
+        anomalies fired (so the frontend can skip rendering the
+        AI-specific charts cleanly).
+        """
+        model_anomalies = [
+            a for a in self.anomalies
+            if (a.metadata or {}).get('family') == 'iforest'
+        ]
+        if not model_anomalies:
+            return {}
+
+        # Category counts (AI: … only). Lets the frontend draw a bar
+        # chart that's specifically about model output, without the
+        # rule-detector counts mixed in like `anomaly_types` has.
+        category_counts: Dict[str, int] = defaultdict(int)
+        for a in model_anomalies:
+            category_counts[a.detection_method] += 1
+
+        # Mean |z| per feature across every model flag. Highest value =
+        # the feature with the strongest average pull on the model — a
+        # natural "feature importance" chart.
+        z_sums: Dict[str, List[float]] = defaultdict(list)
+        for a in model_anomalies:
+            for entry in (a.metadata.get('attribution') or []):
+                z_sums[entry['feature']].append(abs(float(entry['z'])))
+        feature_z_means: Dict[str, float] = {
+            feat: round(sum(zs) / len(zs), 2)
+            for feat, zs in z_sums.items()
+        }
+
+        # 7 fixed buckets covering the detector's 0.60..0.95 confidence
+        # band. Fixed-width buckets so a histogram chart can render
+        # without the frontend deciding bin edges.
+        confidence_buckets: Dict[str, int] = {
+            '0.60-0.65': 0, '0.65-0.70': 0, '0.70-0.75': 0,
+            '0.75-0.80': 0, '0.80-0.85': 0, '0.85-0.90': 0,
+            '0.90-0.95': 0,
+        }
+        for a in model_anomalies:
+            c = a.confidence
+            if c < 0.65:
+                k = '0.60-0.65'
+            elif c < 0.70:
+                k = '0.65-0.70'
+            elif c < 0.75:
+                k = '0.70-0.75'
+            elif c < 0.80:
+                k = '0.75-0.80'
+            elif c < 0.85:
+                k = '0.80-0.85'
+            elif c < 0.90:
+                k = '0.85-0.90'
+            else:
+                k = '0.90-0.95'
+            confidence_buckets[k] += 1
+
+        # Hour-of-day distribution restricted to AI flags only — a
+        # "when does the model fire" view distinct from the frontend's
+        # existing all-data hourly chart.
+        flags_by_hour: List[int] = [0] * 24
+        for a in model_anomalies:
+            original_idx = self.data_indices.get(a.row_index)
+            if original_idx is None:
+                continue
+            row = self.original_data[original_idx]
+            time_str = row.get('event_time') or ''
+            try:
+                hour = int(str(time_str).split(':')[0])
+            except (ValueError, IndexError):
+                continue
+            if 0 <= hour < 24:
+                flags_by_hour[hour] += 1
+
+        # Top 5 subscribers by AI flag count — useful leaderboard chart
+        # on multi-subscriber files. Meaningless on single-subscriber
+        # files (just shows the one subscriber); the frontend can
+        # gracefully hide the card when length <= 1.
+        sub_counts: Dict[str, int] = defaultdict(int)
+        for a in model_anomalies:
+            if a.msisdn:
+                sub_counts[a.msisdn] += 1
+        top_subscribers = [
+            {'msisdn': m, 'flag_count': c}
+            for m, c in sorted(
+                sub_counts.items(), key=lambda x: -x[1]
+            )[:5]
+        ]
+
+        # Pull subscribers_scored / subscribers_skipped from whichever
+        # model detector ran. Defensive — model_detectors list may be
+        # empty (shouldn't happen at this point but cheap to guard).
+        scored = 0
+        skipped = 0
+        for det in self.model_detectors:
+            last = getattr(det, 'last_run_stats', None) or {}
+            scored += int(last.get('subscribers_scored', 0))
+            skipped += int(last.get('subscribers_skipped', 0))
+
+        summary = {
+            'total_flags': len(model_anomalies),
+            'subscribers_scored': scored,
+            'subscribers_skipped': skipped,
+            'category_counts': dict(category_counts),
+            'feature_driver_counts': feature_driver_counts,
+            'feature_z_means': feature_z_means,
+            'confidence_buckets': confidence_buckets,
+            'flags_by_hour': flags_by_hour,
+            'top_subscribers': top_subscribers,
+        }
+
+        # Single-line server log so anyone tailing the API sees the
+        # model output at the end of every run without grepping the
+        # JSON response.
+        logger.info(
+            "IsolationForest summary: %d flags across %d categor%s; "
+            "%d subscribers scored, %d skipped",
+            summary['total_flags'],
+            len(category_counts),
+            'y' if len(category_counts) == 1 else 'ies',
+            scored,
+            skipped,
+        )
+
+        return summary
+
     def export_results(self, format: str = 'json', only_anomalies: bool = True) -> Dict[str, str]:
         """Export all results. CLI-only — the API never calls this."""
         if self.exporter is None:

telecom_anomaly/core/models.py

@@ -107,6 +107,26 @@ class DetectionStatistics:
     # model is finding anomalies in" alongside the rule-detector
     # breakdown.
     model_feature_drivers: Dict[str, int] = field(default_factory=dict)
+    # Consolidated IsolationForest summary — everything the frontend
+    # needs to render iforest-specific charts in one place. Empty when
+    # use_model=False or the model produced zero flags. Keys:
+    #   total_flags             — int, count of iforest anomalies
+    #   subscribers_scored      — int, msisdns that met min_samples
+    #   subscribers_skipped     — int, msisdns below min_samples
+    #   category_counts         — {category: count}, AI: … only
+    #   feature_driver_counts   — {feature: count}, same shape as the
+    #                             top-level model_feature_drivers map
+    #   feature_z_means         — {feature: mean_abs_z}, signal strength
+    #                             per feature across all flags
+    #   confidence_buckets      — {"0.60-0.65": n, …}, 7 buckets
+    #   flags_by_hour           — list[int] of length 24, hourly counts
+    #                             of AI flags only (so the frontend can
+    #                             draw a "when does the AI fire" chart
+    #                             distinct from the all-data hourly
+    #                             distribution it already has)
+    #   top_subscribers         — list[{msisdn, flag_count}], top 5 by
+    #                             AI flag count
+    model_summary: Dict[str, Any] = field(default_factory=dict)
 
     def to_dict(self) -> Dict:
         """Convert to dictionary"""

telecom_anomaly/detection/isolation_forest.py

@@ -83,8 +83,21 @@ class IsolationForestDetector(BaseDetector):
         self.min_samples = getattr(config, 'iforest_min_samples', 50)
         self.random_state = getattr(config, 'iforest_random_state', 42)
         self.n_estimators = getattr(config, 'iforest_n_estimators', 100)
+        # Populated at the end of every `detect()` call. The orchestrator
+        # pulls these into `DetectionStatistics.model_summary` so the
+        # frontend can chart "how much of the data was actually scored"
+        # alongside the per-flag breakdowns.
+        self.last_run_stats: Dict[str, Any] = {
+            'subscribers_scored': 0,
+            'subscribers_skipped': 0,
+        }
 
     def detect(self, data: List[Dict], **kwargs) -> List[AnomalyResult]:
+        # Reset stats so a re-run starts clean.
+        self.last_run_stats = {
+            'subscribers_scored': 0,
+            'subscribers_skipped': 0,
+        }
         if not data:
             return []
 
@@ -94,12 +107,20 @@ class IsolationForestDetector(BaseDetector):
             if msisdn:
                 per_subscriber[msisdn].append(idx)
 
+        scored = 0
+        skipped = 0
         anomalies: List[AnomalyResult] = []
         for msisdn, indices in per_subscriber.items():
             if len(indices) < self.min_samples:
+                skipped += 1
                 continue
+            scored += 1
             anomalies.extend(self._detect_for_subscriber(msisdn, indices, data))
 
+        self.last_run_stats = {
+            'subscribers_scored': scored,
+            'subscribers_skipped': skipped,
+        }
         return anomalies
 
     def _detect_for_subscriber(self, msisdn: str, indices: List[int],