Spaces:
Runtime error
Runtime error
Update memory_drift_diagnostician.py
Browse files
memory_drift_diagnostician.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
import logging
|
| 2 |
import numpy as np
|
| 3 |
-
from typing import Dict, Any, List
|
| 4 |
from agentic_reliability_framework.runtime.agents.base import BaseAgent, AgentSpecialization
|
| 5 |
from ai_event import AIEvent
|
| 6 |
|
|
@@ -14,32 +14,13 @@ class MemoryDriftDiagnosticianAgent(BaseAgent):
|
|
| 14 |
"""
|
| 15 |
|
| 16 |
def __init__(self, history_window: int = 100, zscore_threshold: float = 2.0):
|
| 17 |
-
"""
|
| 18 |
-
Args:
|
| 19 |
-
history_window: Number of recent scores to keep for baseline statistics.
|
| 20 |
-
zscore_threshold: Absolute z‑score above which drift is flagged.
|
| 21 |
-
"""
|
| 22 |
super().__init__(AgentSpecialization.DIAGNOSTICIAN)
|
| 23 |
self.history_window = history_window
|
| 24 |
self.zscore_threshold = zscore_threshold
|
| 25 |
self._retrieval_scores_history: List[float] = []
|
| 26 |
|
| 27 |
async def analyze(self, event: AIEvent) -> Dict[str, Any]:
|
| 28 |
-
"""
|
| 29 |
-
Analyze retrieval scores for drift.
|
| 30 |
-
|
| 31 |
-
Args:
|
| 32 |
-
event: AIEvent containing `retrieval_scores` (list of floats).
|
| 33 |
-
|
| 34 |
-
Returns:
|
| 35 |
-
Dictionary with keys:
|
| 36 |
-
- specialization: str
|
| 37 |
-
- confidence: float (0‑1) based on z‑score magnitude
|
| 38 |
-
- findings: dict with drift detection and statistics
|
| 39 |
-
- recommendations: list of strings if drift detected
|
| 40 |
-
"""
|
| 41 |
try:
|
| 42 |
-
# If no retrieval scores, cannot compute drift
|
| 43 |
if not event.retrieval_scores:
|
| 44 |
return {
|
| 45 |
'specialization': 'ai_memory_drift',
|
|
@@ -48,15 +29,12 @@ class MemoryDriftDiagnosticianAgent(BaseAgent):
|
|
| 48 |
'recommendations': []
|
| 49 |
}
|
| 50 |
|
| 51 |
-
# Current average score
|
| 52 |
current_avg = float(np.mean(event.retrieval_scores))
|
| 53 |
self._retrieval_scores_history.append(current_avg)
|
| 54 |
|
| 55 |
-
# Trim history to window size
|
| 56 |
if len(self._retrieval_scores_history) > self.history_window:
|
| 57 |
self._retrieval_scores_history.pop(0)
|
| 58 |
|
| 59 |
-
# Need at least 10 points for a reliable baseline
|
| 60 |
if len(self._retrieval_scores_history) < 10:
|
| 61 |
return {
|
| 62 |
'specialization': 'ai_memory_drift',
|
|
@@ -70,13 +48,10 @@ class MemoryDriftDiagnosticianAgent(BaseAgent):
|
|
| 70 |
'recommendations': []
|
| 71 |
}
|
| 72 |
|
| 73 |
-
# Historical baseline (excluding current point)
|
| 74 |
historical_avg = float(np.mean(self._retrieval_scores_history[:-1]))
|
| 75 |
-
historical_std = float(np.std(self._retrieval_scores_history[:-1])) + 1e-6
|
| 76 |
z_score = (current_avg - historical_avg) / historical_std
|
| 77 |
drift_detected = abs(z_score) > self.zscore_threshold
|
| 78 |
-
|
| 79 |
-
# Confidence derived from z‑score magnitude (capped at 1.0)
|
| 80 |
confidence = min(1.0, abs(z_score) / 5.0)
|
| 81 |
|
| 82 |
return {
|
|
@@ -94,7 +69,6 @@ class MemoryDriftDiagnosticianAgent(BaseAgent):
|
|
| 94 |
"Update context window"
|
| 95 |
] if drift_detected else []
|
| 96 |
}
|
| 97 |
-
|
| 98 |
except Exception as e:
|
| 99 |
logger.error(f"MemoryDriftDiagnostician error: {e}", exc_info=True)
|
| 100 |
return {
|
|
|
|
| 1 |
import logging
|
| 2 |
import numpy as np
|
| 3 |
+
from typing import Dict, Any, List
|
| 4 |
from agentic_reliability_framework.runtime.agents.base import BaseAgent, AgentSpecialization
|
| 5 |
from ai_event import AIEvent
|
| 6 |
|
|
|
|
| 14 |
"""
|
| 15 |
|
| 16 |
def __init__(self, history_window: int = 100, zscore_threshold: float = 2.0):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 17 |
super().__init__(AgentSpecialization.DIAGNOSTICIAN)
|
| 18 |
self.history_window = history_window
|
| 19 |
self.zscore_threshold = zscore_threshold
|
| 20 |
self._retrieval_scores_history: List[float] = []
|
| 21 |
|
| 22 |
async def analyze(self, event: AIEvent) -> Dict[str, Any]:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 23 |
try:
|
|
|
|
| 24 |
if not event.retrieval_scores:
|
| 25 |
return {
|
| 26 |
'specialization': 'ai_memory_drift',
|
|
|
|
| 29 |
'recommendations': []
|
| 30 |
}
|
| 31 |
|
|
|
|
| 32 |
current_avg = float(np.mean(event.retrieval_scores))
|
| 33 |
self._retrieval_scores_history.append(current_avg)
|
| 34 |
|
|
|
|
| 35 |
if len(self._retrieval_scores_history) > self.history_window:
|
| 36 |
self._retrieval_scores_history.pop(0)
|
| 37 |
|
|
|
|
| 38 |
if len(self._retrieval_scores_history) < 10:
|
| 39 |
return {
|
| 40 |
'specialization': 'ai_memory_drift',
|
|
|
|
| 48 |
'recommendations': []
|
| 49 |
}
|
| 50 |
|
|
|
|
| 51 |
historical_avg = float(np.mean(self._retrieval_scores_history[:-1]))
|
| 52 |
+
historical_std = float(np.std(self._retrieval_scores_history[:-1])) + 1e-6
|
| 53 |
z_score = (current_avg - historical_avg) / historical_std
|
| 54 |
drift_detected = abs(z_score) > self.zscore_threshold
|
|
|
|
|
|
|
| 55 |
confidence = min(1.0, abs(z_score) / 5.0)
|
| 56 |
|
| 57 |
return {
|
|
|
|
| 69 |
"Update context window"
|
| 70 |
] if drift_detected else []
|
| 71 |
}
|
|
|
|
| 72 |
except Exception as e:
|
| 73 |
logger.error(f"MemoryDriftDiagnostician error: {e}", exc_info=True)
|
| 74 |
return {
|