fix AI model

check_attribution.py

@@ -0,0 +1,51 @@
+"""One-off: load a CSV via the real reader, run the detector with
+use_model=True, and print:
+  - the new categorised method names + per-row attribution,
+  - the aggregate `model_feature_drivers` map from get_statistics()
+    (the shape the frontend will chart).
+"""
+import sys
+from pathlib import Path
+sys.path.insert(0, str(Path(__file__).parent))
+
+from telecom_anomaly.core.detector import TelecomAnomalyDetector
+from telecom_anomaly.core.models import TelecomConfig
+
+CSV = (
+    r"C:\my_project\work\GH_Police\main project\backend"
+    r"\telecom_anomaly_detection\data\strictly_cleaned_data"
+    r"\591296805 calls (1)_strictly_cleaned.csv"
+)
+
+cfg = TelecomConfig()
+det = TelecomAnomalyDetector(cfg)
+det.load_data([CSV])
+det.detect_anomalies(use_model=True)
+
+stats = det.get_statistics()
+print(f"\nTotal anomalies: {stats.total_anomalies_detected}")
+print(f"Rows with anomalies: {stats.rows_with_anomalies}")
+print(f"\nanomaly_types (rule + model categories combined):")
+for k, v in sorted(stats.anomaly_types.items(), key=lambda x: -x[1]):
+    print(f"  {v:4d}  {k}")
+
+print(f"\nmodel_feature_drivers (top-driver counts across all AI flags):")
+for k, v in sorted(stats.model_feature_drivers.items(), key=lambda x: -x[1]):
+    print(f"  {v:4d}  {k}")
+
+print(f"\nSample AI-flagged rows with category + reason + structured attribution:")
+shown = 0
+for orig_idx, methods in det.row_anomaly_map.items():
+    ai = [(i, m) for i, m in enumerate(methods) if m.startswith('AI:')]
+    if not ai or shown >= 5:
+        continue
+    i, method = ai[0]
+    reason = det.row_anomaly_reasons[orig_idx][i]
+    attribution = det.row_anomaly_attribution[orig_idx][i]
+    row = det.original_data[orig_idx]
+    print(f"\n  row {orig_idx}: {row.get('event_date')} {row.get('event_time')} "
+          f"dur={row.get('duration')}")
+    print(f"    method   : {method}")
+    print(f"    reason   : {reason}")
+    print(f"    attribution: {attribution}")
+    shown += 1

profile_iforest.py

@@ -0,0 +1,197 @@
+"""
+One-off profiler for the two strictly-cleaned CSVs to inform IsolationForest
+tuning. Reports schema, per-subscriber row counts, feature distributions,
+hour-of-day histograms, and the actual flag rate the current model produces
+on each file at the current defaults.
+
+Run from `teleprocessinggh/`:
+    .venv\\Scripts\\python.exe profile_iforest.py
+"""
+
+import os
+import sys
+from collections import Counter, defaultdict
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+
+# Make `telecom_anomaly.*` imports resolve when run from this folder.
+sys.path.insert(0, str(Path(__file__).parent))
+
+from telecom_anomaly.core.models import TelecomConfig  # noqa: E402
+from telecom_anomaly.detection.isolation_forest import (  # noqa: E402
+    IsolationForestDetector,
+)
+from telecom_anomaly.validation.validator import DataValidator  # noqa: E402
+
+
+CSV_PATHS = [
+    r"C:\my_project\work\GH_Police\main project\backend\telecom_anomaly_detection"
+    r"\data\strictly_cleaned_data\246060920_calls_strictly_cleaned.csv",
+    r"C:\my_project\work\GH_Police\main project\backend\telecom_anomaly_detection"
+    r"\data\strictly_cleaned_data\591296805 calls (1)_strictly_cleaned.csv",
+]
+
+
+def hr(c="-", n=72):
+    print(c * n)
+
+
+def profile(path):
+    name = os.path.basename(path)
+    print()
+    hr("=")
+    print(f"FILE: {name}")
+    hr("=")
+
+    df = pd.read_csv(path, dtype=str, keep_default_na=False, low_memory=False)
+    print(f"Rows: {len(df):,}   Columns: {len(df.columns)}")
+    print(f"Headers: {list(df.columns)}")
+
+    v = DataValidator()
+    # Derive msisdn (matches the reader's logic).
+    def _msisdn(r):
+        explicit = r.get("msisdn", "")
+        if v.validate_msisdn(explicit):
+            return explicit
+        return v.derive_msisdn(
+            r.get("call_direction", ""),
+            r.get("calling_no", ""),
+            r.get("called_no", ""),
+        )
+
+    df["_msisdn"] = df.apply(_msisdn, axis=1)
+    valid = df[df["_msisdn"] != ""].copy()
+    print(f"Rows with derivable msisdn: {len(valid):,} / {len(df):,}")
+
+    # Per-subscriber row counts
+    sub_counts = valid["_msisdn"].value_counts()
+    print(f"Distinct subscribers: {len(sub_counts):,}")
+    print(f"  rows-per-subscriber  min={sub_counts.min()}  "
+          f"p50={int(sub_counts.median())}  p90={int(sub_counts.quantile(0.9))}  "
+          f"max={sub_counts.max()}")
+    eligible = sub_counts[sub_counts >= 50]
+    print(f"Subscribers with >=50 rows (eligible for IForest): "
+          f"{len(eligible)} ({eligible.sum():,} rows)")
+    print(f"Top 5 subscribers by row count:")
+    for m, c in sub_counts.head(5).items():
+        print(f"  {m}  ->  {c:,} rows")
+
+    # Categorical distributions
+    print()
+    print("usage_type:")
+    for k, c in valid["usage_type"].value_counts().head(10).items():
+        print(f"  {k or '<empty>':20s}  {c:6,}")
+    print("call_direction:")
+    for k, c in valid["call_direction"].value_counts().head(10).items():
+        # call_direction often contains a newline; collapse for readability
+        kk = " ".join(str(k).split())
+        print(f"  {kk or '<empty>':20s}  {c:6,}")
+
+    # Duration distribution
+    dur = pd.to_numeric(valid["duration"], errors="coerce").fillna(0)
+    print()
+    print(f"duration (sec)  min={dur.min():.0f}  "
+          f"p50={dur.quantile(0.5):.0f}  p90={dur.quantile(0.9):.0f}  "
+          f"p99={dur.quantile(0.99):.0f}  max={dur.max():.0f}")
+    print(f"  duration < 3s   : {(dur < 3).sum():,}   "
+          f"(flash-call territory)")
+    print(f"  duration > 1hr  : {(dur > 3600).sum():,}   "
+          f"(extreme-call territory)")
+
+    # Hour distribution
+    def _hour(t):
+        try:
+            return int(str(t).split(":")[0])
+        except Exception:
+            return -1
+    valid["_hour"] = valid["event_time"].apply(_hour)
+    hist = valid[valid["_hour"] >= 0]["_hour"].value_counts().sort_index()
+    print()
+    print("Hour-of-day histogram (counts):")
+    for h in range(24):
+        count = int(hist.get(h, 0))
+        bar = "#" * min(60, count // max(1, sub_counts.sum() // 800))
+        marker = " *night*" if (h >= 23 or h < 5) else ""
+        print(f"  {h:02d}:00  {count:6,}  {bar}{marker}")
+    night = valid[(valid["_hour"] >= 23) | ((valid["_hour"] >= 0) & (valid["_hour"] < 5))]
+    print(f"Night-hour rows (23:00-04:59): {len(night):,} "
+          f"({100*len(night)/max(1,len(valid)):.1f}%)")
+
+    # Coordinate coverage
+    lat = pd.to_numeric(valid["latitude"], errors="coerce")
+    lon = pd.to_numeric(valid["longitude"], errors="coerce")
+    print()
+    print(f"latitude  present: {lat.notna().sum():,} "
+          f"({100*lat.notna().sum()/len(valid):.1f}%)")
+    print(f"longitude present: {lon.notna().sum():,} "
+          f"({100*lon.notna().sum()/len(valid):.1f}%)")
+
+    # Region / district cardinality
+    print(f"distinct regions  : {valid['region'].nunique()}")
+    print(f"distinct districts: {valid['district'].nunique()}")
+    if 'imei' in valid.columns:
+        print(f"distinct IMEIs    : {valid['imei'].nunique()}")
+
+    # ----- Run IForest at multiple contamination levels -----
+    print()
+    print("IsolationForest flag rates per contamination (current min_samples=50):")
+    # Build records list in the shape the detector expects.
+    records = []
+    for _, r in valid.iterrows():
+        records.append({
+            "msisdn": r["_msisdn"],
+            "duration": pd.to_numeric(r.get("duration"), errors="coerce") or 0.0,
+            "event_time": r.get("event_time", ""),
+            "is_night": (_hour(r.get("event_time")) >= 23
+                         or 0 <= _hour(r.get("event_time")) < 5),
+            "usage_type": r.get("usage_type", ""),
+            "call_direction": r.get("call_direction", ""),
+            "latitude": float(r["latitude"])
+            if r.get("latitude") not in ("", None) and pd.notna(
+                pd.to_numeric(r["latitude"], errors="coerce")) else None,
+            "longitude": float(r["longitude"])
+            if r.get("longitude") not in ("", None) and pd.notna(
+                pd.to_numeric(r["longitude"], errors="coerce")) else None,
+            "region": r.get("region", ""),
+            "district": r.get("district", ""),
+        })
+
+    for contamination in ["auto", 0.10, 0.05, 0.02, 0.01]:
+        cfg = TelecomConfig()
+        cfg.iforest_contamination = contamination
+        cfg.iforest_min_samples = 50
+        det = IsolationForestDetector(cfg)
+        flagged = det.detect(records)
+        scored_subs = sum(
+            1 for s, idxs in
+            _group_subscribers(records).items() if len(idxs) >= 50
+        )
+        scored_rows = sum(
+            len(idxs) for s, idxs in
+            _group_subscribers(records).items() if len(idxs) >= 50
+        )
+        rate = 100 * len(flagged) / max(1, scored_rows)
+        print(f"  contamination={str(contamination):>5}  "
+              f"flagged={len(flagged):4}  "
+              f"of scored_rows={scored_rows:5}  "
+              f"({rate:5.2f}%)   "
+              f"across {scored_subs} subscribers")
+
+
+def _group_subscribers(records):
+    g = defaultdict(list)
+    for i, r in enumerate(records):
+        m = r.get("msisdn", "")
+        if m:
+            g[m].append(i)
+    return g
+
+
+if __name__ == "__main__":
+    for path in CSV_PATHS:
+        if not os.path.exists(path):
+            print(f"Missing: {path}")
+            continue
+        profile(path)

telecom_anomaly/api.py

@@ -222,11 +222,25 @@ def _run_analysis(
         methods = detector.row_anomaly_map.get(idx, [])
         if run_anomaly_detection and not methods:
             continue
+        reasons = detector.row_anomaly_reasons.get(idx, [])
+        attributions = detector.row_anomaly_attribution.get(idx, [])
         record = {field: row.get(field, "") for field in REQUESTED_FIELDS}
         record.update({
             "has_anomaly": "YES" if methods else "NO",
             "anomaly_count": len(methods),
             "anomaly_methods": "; ".join(methods),
+            # Index-aligned with anomaly_methods. Currently only the iforest
+            # detector populates a non-empty reason; rule detectors leave
+            # their slot blank. Empty entries are dropped so the joined
+            # string stays readable.
+            "anomaly_reasons": "; ".join(r for r in reasons if r),
+            # Structured per-method attribution. Each entry is the top-k
+            # {feature, z, value, direction} dicts the detector produced.
+            # Rule detectors contribute empty lists; iforest contributes
+            # 1-3 entries per flag. The frontend can render these as
+            # per-row bar charts ("why was this flagged"), or aggregate
+            # them client-side for a global feature-importance view.
+            "anomaly_attribution": attributions,
         })
         full_records.append(record)
 

telecom_anomaly/core/detector.py

@@ -80,6 +80,18 @@ class TelecomAnomalyDetector:
         # Anomaly tracking
         self.rows_with_anomalies: Set[int] = set()
         self.row_anomaly_map: Dict[int, List[str]] = defaultdict(list)
+        # Per-row "reason" strings, parallel to row_anomaly_map (index i in
+        # the value list corresponds to method i). Empty string when the
+        # method didn't supply a reason (rule detectors today). The iforest
+        # detector populates this from anomaly.metadata['reason'].
+        self.row_anomaly_reasons: Dict[int, List[str]] = defaultdict(list)
+        # Per-row structured attribution, parallel to row_anomaly_map.
+        # Each entry is a list of {feature, z, value, direction} dicts
+        # (or an empty list for methods that didn't compute attribution —
+        # i.e. the rule detectors today). Lets the frontend draw per-row
+        # bar charts of which features drove a flag.
+        self.row_anomaly_attribution: Dict[int, List[List[Dict[str, Any]]]] = \
+            defaultdict(list)
         self.subscriber_anomalies: Dict[str, Dict] = defaultdict(
             lambda: {'total': 0, 'methods': set()}
         )
@@ -270,7 +282,21 @@ class TelecomAnomalyDetector:
                 original_idx = self.data_indices.get(anomaly.row_index)
                 if original_idx is not None:
                     self.row_anomaly_map[original_idx].append(anomaly.detection_method)
-            
+                    # Mirror reason + structured attribution in the parallel
+                    # lists. Rule detectors don't currently set
+                    # metadata['reason'] / metadata['attribution'] so they
+                    # fall back to '' / [] to keep all three lists
+                    # index-aligned.
+                    reason = ''
+                    attribution: List[Dict[str, Any]] = []
+                    if anomaly.metadata:
+                        reason = str(anomaly.metadata.get('reason', '') or '')
+                        attribution = list(
+                            anomaly.metadata.get('attribution', []) or []
+                        )
+                    self.row_anomaly_reasons[original_idx].append(reason)
+                    self.row_anomaly_attribution[original_idx].append(attribution)
+
             # Track subscriber anomalies
             if anomaly.msisdn:
                 self.subscriber_anomalies[anomaly.msisdn]['total'] += 1
@@ -279,22 +305,39 @@ class TelecomAnomalyDetector:
     def get_statistics(self) -> DetectionStatistics:
         """Get detection statistics"""
         rows_with_anomalies = len(self.rows_with_anomalies)
-        
+
         # Calculate anomaly rate correctly
         anomaly_rate = (rows_with_anomalies / len(self.data)) * 100 if self.data else 0
-        
+
         # Count anomaly types
         anomaly_types = defaultdict(int)
         for anomaly in self.anomalies:
             anomaly_types[anomaly.detection_method] += 1
-        
+
+        # Aggregate model-flag top-driver counts. Only iforest anomalies
+        # carry a `family='iforest'` metadata marker today, so this naturally
+        # ignores rule detectors. The output is shaped for direct rendering
+        # by a bar chart on the visualization screen.
+        model_feature_drivers: Dict[str, int] = defaultdict(int)
+        for anomaly in self.anomalies:
+            meta = anomaly.metadata or {}
+            if meta.get('family') != 'iforest':
+                continue
+            top = meta.get('top_feature') or ''
+            if not top:
+                # Multivariate-but-no-top-driver flags get bucketed under
+                # an explicit label so the chart still accounts for them.
+                top = '_multivariate'
+            model_feature_drivers[top] += 1
+
         return DetectionStatistics(
             total_rows=len(self.data),
             rows_with_anomalies=rows_with_anomalies,
             total_anomalies_detected=len(self.anomalies),
             anomaly_rate=anomaly_rate,
             unique_subscribers=len(set(a.msisdn for a in self.anomalies if a.msisdn)),
-            anomaly_types=dict(anomaly_types)
+            anomaly_types=dict(anomaly_types),
+            model_feature_drivers=dict(model_feature_drivers),
         )
 
     def export_results(self, format: str = 'json', only_anomalies: bool = True) -> Dict[str, str]:

telecom_anomaly/core/models.py

@@ -101,7 +101,13 @@ class DetectionStatistics:
     anomaly_rate: float
     unique_subscribers: int
     anomaly_types: Dict[str, int]
-    
+    # Per-feature counts of "this feature was the #1 driver" across all
+    # IsolationForest flags. Empty when use_model=False (or no model
+    # anomalies fired). Lets the frontend chart "what features the AI
+    # model is finding anomalies in" alongside the rule-detector
+    # breakdown.
+    model_feature_drivers: Dict[str, int] = field(default_factory=dict)
+
     def to_dict(self) -> Dict:
         """Convert to dictionary"""
         return asdict(self)

telecom_anomaly/detection/isolation_forest.py

@@ -10,10 +10,16 @@ single rule fires).
 Trained per-msisdn so each subscriber's "normal" is defined relative to
 their own history. Subscribers below `min_samples_for_model` rows are
 skipped to avoid overfitting noise.
+
+Each flagged row carries a per-feature attribution (top-3 features that
+pulled it furthest from the subscriber's mean, in z-score terms) so the
+caller can show *why* the row was flagged — the rule detectors give
+single-column reasons, this gives multivariate ones.
 """
 
 from collections import Counter, defaultdict
-from typing import List, Dict, Any
+from datetime import datetime, time as _time
+from typing import Any, Dict, List, Tuple
 
 import numpy as np
 from sklearn.ensemble import IsolationForest
@@ -35,11 +41,45 @@ class IsolationForestDetector(BaseDetector):
         'longitude',
         'region_freq',
         'district_freq',
+        'imei_freq',
+        'time_since_last_call_sec',
     ]
 
+    # Maps each feature to the category label used as the row's
+    # detection_method. Picking the *category* of the top-z feature lets the
+    # frontend donut chart show the model's findings broken down the same
+    # way the rule detectors are ("DurationDetector", "SMSDetector", …)
+    # rather than collapsing every model flag into a single "iforest" slice.
+    _FEATURE_CATEGORY = {
+        'duration': 'AI: Duration Outlier',
+        'hour_of_day': 'AI: Time Pattern',
+        'is_night': 'AI: Time Pattern',
+        'is_sms': 'AI: Usage Pattern',
+        'direction_outgoing': 'AI: Usage Pattern',
+        'latitude': 'AI: Location Outlier',
+        'longitude': 'AI: Location Outlier',
+        'region_freq': 'AI: Location Outlier',
+        'district_freq': 'AI: Location Outlier',
+        'imei_freq': 'AI: Device Switch',
+        'time_since_last_call_sec': 'AI: Bursty Pattern',
+    }
+    _DEFAULT_CATEGORY = 'AI: Multivariate'
+
+    # Z-score below which a feature is too "normal" to be worth surfacing in
+    # the per-row reason string. Set conservatively so reasons stay focused
+    # on what actually drove the flag.
+    _ATTRIBUTION_MIN_ABS_Z = 1.0
+
+    # Max number of feature attributions to surface per flagged row.
+    _ATTRIBUTION_TOP_K = 3
+
     def __init__(self, config: TelecomConfig):
         super().__init__(config)
-        self.contamination = getattr(config, 'iforest_contamination', 'auto')
+        # Default contamination tightened from sklearn's 'auto' (which flags
+        # ~10–17% of rows on the canonical CSVs) down to 2%, matching the
+        # tuning recommendation in planing.md. Override via TelecomConfig
+        # if you want the legacy 'auto' behaviour back.
+        self.contamination = getattr(config, 'iforest_contamination', 0.02)
         self.min_samples = getattr(config, 'iforest_min_samples', 50)
         self.random_state = getattr(config, 'iforest_random_state', 42)
         self.n_estimators = getattr(config, 'iforest_n_estimators', 100)
@@ -88,6 +128,16 @@ class IsolationForestDetector(BaseDetector):
         s_min, s_max = float(scores.min()), float(scores.max())
         denom = (s_max - s_min) or 1.0
 
+        # Per-feature z-scores across the subscriber's data. Drives the
+        # attribution / "reason" strings on each flagged row.
+        means = features.mean(axis=0)
+        stds = features.std(axis=0)
+        # Avoid div-by-zero for features with no variance in this subscriber
+        # (e.g. a subscriber whose every call is is_sms=0); their z-scores
+        # become 0, so they won't appear in the top-k.
+        stds_safe = np.where(stds > 1e-9, stds, 1.0)
+        z_all = (features - means) / stds_safe
+
         results: List[AnomalyResult] = []
         for local_idx, global_idx in enumerate(indices):
             if predictions[local_idx] != -1:
@@ -95,11 +145,26 @@ class IsolationForestDetector(BaseDetector):
             raw_score = float(scores[local_idx])
             normalized = (s_max - raw_score) / denom            # 0..1
             confidence = round(0.6 + 0.35 * normalized, 4)      # 0.60..0.95
+
+            attribution, reason = self._attribution_for_row(
+                z_all[local_idx], features[local_idx], means
+            )
+
+            # Category-as-method name: e.g. "AI: Location Outlier" instead
+            # of one undifferentiated "iforest" slice. The category is
+            # driven by the top-z feature in the attribution; rows with no
+            # attribution (all |z| < threshold) fall back to "AI:
+            # Multivariate".
+            top_feature = attribution[0]['feature'] if attribution else None
+            method_name = self._FEATURE_CATEGORY.get(
+                top_feature, self._DEFAULT_CATEGORY
+            ) if top_feature else self._DEFAULT_CATEGORY
+
             results.append(self._create_anomaly(
                 row_index=global_idx,
                 column_name='multivariate',
                 value=raw_score,
-                method='iforest',
+                method=method_name,
                 score=raw_score,
                 threshold=float(model.offset_),
                 confidence=confidence,
@@ -107,13 +172,73 @@ class IsolationForestDetector(BaseDetector):
                 metadata={
                     'features': self.FEATURE_COLUMNS,
                     'subscriber_samples': len(indices),
+                    'attribution': attribution,
+                    'reason': reason,
+                    # Family marker so downstream code can tell "any AI
+                    # flag" apart from rule-detector flags even though
+                    # detection_method now varies.
+                    'family': 'iforest',
+                    'top_feature': top_feature or '',
                 },
             ))
         return results
 
+    def _attribution_for_row(
+        self,
+        z_row: np.ndarray,
+        feature_row: np.ndarray,
+        feature_means: np.ndarray,
+    ) -> Tuple[List[Dict[str, Any]], str]:
+        """
+        Pick the top-k features by |z|, build a human-readable reason.
+
+        Returns:
+          attribution: list of {feature, z, value, direction} dicts
+              (machine-readable; ends up in metadata so the visualisation
+              screen can render bars / chips per feature).
+          reason: short string like
+              "duration high (+3.2σ); imei_freq low (-2.1σ)"
+              suitable for inline display alongside the rule detectors'
+              single-column reasons.
+        """
+        abs_z = np.abs(z_row)
+        order = np.argsort(-abs_z)
+        picks: List[Tuple[int, float]] = []
+        for k in order:
+            if len(picks) >= self._ATTRIBUTION_TOP_K:
+                break
+            if abs_z[k] < self._ATTRIBUTION_MIN_ABS_Z:
+                break
+            picks.append((int(k), float(z_row[k])))
+
+        attribution: List[Dict[str, Any]] = []
+        reason_parts: List[str] = []
+        for k, z in picks:
+            name = self.FEATURE_COLUMNS[k]
+            value = float(feature_row[k])
+            direction = 'high' if z >= 0 else 'low'
+            attribution.append({
+                'feature': name,
+                'z': round(z, 2),
+                'value': round(value, 3),
+                'direction': direction,
+            })
+            reason_parts.append(
+                f"{name} {direction} ({'+' if z >= 0 else '-'}{abs(z):.1f}σ)"
+            )
+
+        # If literally nothing stood out (all |z| < 1) we still want the row
+        # to have a reason — the score itself was anomalous in the model's
+        # joint distribution, just no single feature dominated.
+        reason = '; '.join(reason_parts) if reason_parts else \
+            'multivariate combination outside subscriber norm'
+        return attribution, reason
+
     def _build_features(self, rows: List[Dict[str, Any]]) -> np.ndarray:
         region_freq = self._frequency_map(rows, 'region')
         district_freq = self._frequency_map(rows, 'district')
+        imei_freq = self._frequency_map(rows, 'imei')
+        gaps = self._time_since_last_map(rows)
 
         lat_values = [r.get('latitude') for r in rows if r.get('latitude') is not None]
         lon_values = [r.get('longitude') for r in rows if r.get('longitude') is not None]
@@ -121,7 +246,7 @@ class IsolationForestDetector(BaseDetector):
         lon_default = float(np.median(lon_values)) if lon_values else 0.0
 
         features = []
-        for row in rows:
+        for i, row in enumerate(rows):
             duration = float(row.get('duration') or 0.0)
             hour = self._hour_of_day(row.get('event_time', ''))
             is_night = 1 if row.get('is_night') else 0
@@ -134,6 +259,7 @@ class IsolationForestDetector(BaseDetector):
             lon = float(lon) if lon is not None else lon_default
             region = str(row.get('region') or '')
             district = str(row.get('district') or '')
+            imei = str(row.get('imei') or '')
 
             features.append([
                 duration,
@@ -145,6 +271,8 @@ class IsolationForestDetector(BaseDetector):
                 lon,
                 region_freq.get(region, 0.0),
                 district_freq.get(district, 0.0),
+                imei_freq.get(imei, 0.0),
+                gaps[i],
             ])
 
         return np.asarray(features, dtype=np.float64)
@@ -160,10 +288,44 @@ class IsolationForestDetector(BaseDetector):
         total = sum(counts.values()) or 1
         return {value: count / total for value, count in counts.items()}
 
+    @staticmethod
+    def _time_since_last_map(rows: List[Dict]) -> List[float]:
+        """
+        Seconds since this subscriber's previous call (in chronological
+        order). The first chronological row gets 0. Rows without a
+        parseable datetime also get 0 (they sort to the front but the
+        IForest sees them as "unknown gap" — harmless given the rest of
+        the feature signal).
+
+        Returns a list aligned with `rows`' original order so the feature
+        matrix stays index-stable.
+        """
+        n = len(rows)
+        result = [0.0] * n
+
+        with_dt: List[Tuple[int, datetime]] = []
+        for i, r in enumerate(rows):
+            dt = r.get('datetime')
+            if isinstance(dt, datetime):
+                with_dt.append((i, dt))
+
+        if not with_dt:
+            return result
+
+        with_dt.sort(key=lambda x: x[1])
+        prev_dt: datetime = with_dt[0][1]
+        for orig_idx, dt in with_dt[1:]:
+            delta = (dt - prev_dt).total_seconds()
+            # Clamp tiny negative drift (shouldn't happen post-sort but be
+            # safe) and cap absurd gaps so a single multi-day silence
+            # doesn't dominate the std.
+            result[orig_idx] = max(0.0, float(delta))
+            prev_dt = dt
+        return result
+
     @staticmethod
     def _hour_of_day(time_str) -> int:
         try:
-            from datetime import time as _time
             if isinstance(time_str, _time):
                 return int(time_str.hour)
             return int(str(time_str).split(':')[0])