contextflow-rl / app /agents /behavioral_agent.py
namish10's picture
Upload app/agents/behavioral_agent.py with huggingface_hub
690ed7d verified
"""
Behavioral Agent - Updated without camera eye tracking
Tracks behavioral signals through:
- Mouse movements (hesitation, click patterns)
- Scroll behavior (reversals, speed)
- Keyboard patterns (typing speed, corrections)
- Time patterns (learning duration, break frequency)
- Hand gestures (trained by user)
- Page focus/blur events
- Selection patterns
"""
import time
from typing import Dict, List, Any, Optional
from dataclasses import dataclass, field
from datetime import datetime
from collections import deque
import numpy as np
@dataclass
class MouseData:
"""Mouse movement data"""
x: float
y: float
velocity: float
acceleration: float
click_count: int
selection_count: int
right_click: bool
timestamp: datetime
@dataclass
class ScrollData:
"""Scroll behavior data"""
scroll_up_count: int
scroll_down_count: int
reversal_count: int
avg_scroll_speed: float
max_scroll_speed: float
direction_changes: int
timestamp: datetime
@dataclass
class KeyboardData:
"""Keyboard activity data"""
key_presses: int
backspaces: int
typing_speed: float
corrections: int
pause_duration: float
timestamp: datetime
@dataclass
class FocusData:
"""Tab/window focus data"""
is_focused: bool
visible_duration: float
hidden_duration: float
tab_switches: int
timestamp: datetime
@dataclass
class BehavioralSignal:
"""Aggregated behavioral signal"""
signal_type: str
value: float
timestamp: datetime
source: str
metadata: Dict = field(default_factory=dict)
@dataclass
class TabActivity:
"""Tab activity tracking"""
tab_id: str
url: str
title: str
active_time: float = 0
scroll_depth: float = 0
clicks: int = 0
keystrokes: int = 0
focus_count: int = 0
class BehavioralAgent:
"""
Agent that tracks behavioral signals without camera access.
Signal Sources:
1. Mouse - movements, clicks, selections
2. Scroll - patterns, reversals, speed
3. Keyboard - typing, corrections, pauses
4. Focus - tab switches, visibility
5. Hand Gestures - trained by user
"""
def __init__(self, user_id: str, config: Optional[Dict] = None):
self.user_id = user_id
self.config = config or {}
self.signal_buffer = deque(maxlen=100)
self.mouse_buffer = deque(maxlen=50)
self.scroll_buffer = deque(maxlen=50)
self.keyboard_buffer = deque(maxlen=50)
self.focus_buffer = deque(maxlen=50)
self.gesture_buffer = deque(maxlen=20)
self.baseline_established = False
self.baseline_metrics = {}
self.tab_tracking: Dict[str, TabActivity] = {}
self.current_tab: Optional[str] = None
self.session_start = datetime.now()
self.total_active_time = 0
self.total_break_time = 0
self.last_mouse_move = None
self.last_scroll_time = None
self.last_keypress = None
self.scroll_direction = 1
def add_mouse_data(self, data: Dict) -> MouseData:
"""Add mouse movement data"""
mouse_data = MouseData(
x=data.get('x', 0),
y=data.get('y', 0),
velocity=data.get('velocity', 0),
acceleration=data.get('acceleration', 0),
click_count=data.get('click_count', 0),
selection_count=data.get('selection_count', 0),
right_click=data.get('right_click', False),
timestamp=datetime.now()
)
self.mouse_buffer.append(mouse_data)
if self.last_mouse_move:
dt = (mouse_data.timestamp - self.last_mouse_move).total_seconds()
if dt > 2:
signal = BehavioralSignal(
signal_type='hesitation',
value=min(dt / 10, 1.0),
timestamp=mouse_data.timestamp,
source='mouse',
metadata={'duration': dt}
)
self.signal_buffer.append(signal)
self.last_mouse_move = mouse_data.timestamp
return mouse_data
def add_scroll_data(self, data: Dict) -> ScrollData:
"""Add scroll behavior data"""
direction = data.get('direction', 'down')
if self.last_scroll_time:
dt = (datetime.now() - self.last_scroll_time).total_seconds()
if dt > 0.5 and direction != self.scroll_direction:
reversal = BehavioralSignal(
signal_type='scroll_reversal',
value=0.5,
timestamp=datetime.now(),
source='scroll'
)
self.signal_buffer.append(reversal)
self.scroll_direction = 1 if direction == 'down' else -1
self.last_scroll_time = datetime.now()
scroll_data = ScrollData(
scroll_up_count=data.get('scroll_up_count', 0),
scroll_down_count=data.get('scroll_down_count', 0),
reversal_count=data.get('reversal_count', 0),
avg_scroll_speed=data.get('avg_scroll_speed', 0),
max_scroll_speed=data.get('max_scroll_speed', 0),
direction_changes=data.get('direction_changes', 0),
timestamp=datetime.now()
)
self.scroll_buffer.append(scroll_data)
return scroll_data
def add_keyboard_data(self, data: Dict) -> KeyboardData:
"""Add keyboard activity data"""
keyboard_data = KeyboardData(
key_presses=data.get('key_presses', 0),
backspaces=data.get('backspaces', 0),
typing_speed=data.get('typing_speed', 0),
corrections=data.get('corrections', 0),
pause_duration=data.get('pause_duration', 0),
timestamp=datetime.now()
)
self.keyboard_buffer.append(keyboard_data)
if keyboard_data.pause_duration > 5:
signal = BehavioralSignal(
signal_type='typing_pause',
value=min(keyboard_data.pause_duration / 30, 1.0),
timestamp=keyboard_data.timestamp,
source='keyboard',
metadata={'pause_duration': keyboard_data.pause_duration}
)
self.signal_buffer.append(signal)
return keyboard_data
def add_focus_data(self, data: Dict) -> FocusData:
"""Add tab/window focus data"""
focus_data = FocusData(
is_focused=data.get('is_focused', True),
visible_duration=data.get('visible_duration', 0),
hidden_duration=data.get('hidden_duration', 0),
tab_switches=data.get('tab_switches', 0),
timestamp=datetime.now()
)
self.focus_buffer.append(focus_data)
if not focus_data.is_focused:
signal = BehavioralSignal(
signal_type='unfocused',
value=0.8,
timestamp=focus_data.timestamp,
source='focus'
)
self.signal_buffer.append(signal)
return focus_data
def add_tab_data(self, data: Dict):
"""Track tab activity"""
tab_id = data.get('tab_id')
if not tab_id:
return
if tab_id not in self.tab_tracking:
self.tab_tracking[tab_id] = TabActivity(
tab_id=tab_id,
url=data.get('url', ''),
title=data.get('title', '')
)
tab = self.tab_tracking[tab_id]
if 'scroll_depth' in data:
tab.scroll_depth = data['scroll_depth']
if 'clicks' in data:
tab.clicks += data['clicks']
if 'keystrokes' in data:
tab.keystrokes += data['keystrokes']
if 'focus' in data and data['focus']:
tab.focus_count += 1
def add_gesture_signal(self, gesture_signal: Dict):
"""Add hand gesture signal"""
signal = BehavioralSignal(
signal_type=gesture_signal.get('signal_type', 'unknown'),
value=gesture_signal.get('confidence', 0.5),
timestamp=datetime.fromisoformat(gesture_signal.get('timestamp', datetime.now().isoformat())),
source='gesture',
metadata={
'gesture_name': gesture_signal.get('gesture_name', ''),
'raw_confidence': gesture_signal.get('raw_confidence', 0),
'description': gesture_signal.get('description', '')
}
)
self.gesture_buffer.append(signal)
self.signal_buffer.append(signal)
def process_signals(self, data: Dict) -> List[BehavioralSignal]:
"""Process all incoming behavioral signals"""
signals = []
if 'mouse' in data:
mouse_data = self.add_mouse_data(data['mouse'])
signal = self._analyze_mouse_pattern(mouse_data)
signals.append(signal)
if 'scroll' in data:
scroll_data = self.add_scroll_data(data['scroll'])
signal = self._analyze_scroll_behavior(scroll_data)
signals.append(signal)
if 'keyboard' in data:
keyboard_data = self.add_keyboard_data(data['keyboard'])
signal = self._analyze_keyboard_pattern(keyboard_data)
signals.append(signal)
if 'focus' in data:
focus_data = self.add_focus_data(data['focus'])
signal = self._analyze_focus(focus_data)
signals.append(signal)
if 'gesture' in data:
self.add_gesture_signal(data['gesture'])
if 'tab' in data:
self.add_tab_data(data['tab'])
return signals
def _analyze_mouse_pattern(self, data: MouseData) -> BehavioralSignal:
"""Analyze mouse patterns"""
hesitation_threshold = 50
velocity_threshold = 100
hesitation_score = min(data.acceleration / hesitation_threshold, 1.0)
velocity_score = min(data.velocity / velocity_threshold, 1.0)
combined = (hesitation_score * 0.6 + velocity_score * 0.4)
if data.right_click:
combined = max(combined, 0.6)
return BehavioralSignal(
signal_type='mouse_activity',
value=combined,
timestamp=data.timestamp,
source='mouse',
metadata={
'hesitation': hesitation_score,
'velocity': velocity_score,
'clicks': data.click_count
}
)
def _analyze_scroll_behavior(self, data: ScrollData) -> BehavioralSignal:
"""Analyze scroll behavior"""
if data.scroll_up_count + data.scroll_down_count == 0:
return BehavioralSignal(
signal_type='scroll_inactive',
value=0.0,
timestamp=data.timestamp,
source='scroll'
)
reversal_ratio = data.reversal_count / max(
data.scroll_up_count + data.scroll_down_count, 1
)
reversal_score = min(reversal_ratio * 2, 1.0)
speed_variation = 0
if data.max_scroll_speed > 0:
speed_variation = abs(data.avg_scroll_speed - data.max_scroll_speed) / data.max_scroll_speed
combined = reversal_score * 0.7 + speed_variation * 0.3
return BehavioralSignal(
signal_type='scroll_pattern',
value=combined,
timestamp=data.timestamp,
source='scroll',
metadata={
'reversal_ratio': reversal_ratio,
'speed_variation': speed_variation
}
)
def _analyze_keyboard_pattern(self, data: KeyboardData) -> BehavioralSignal:
"""Analyze keyboard patterns"""
if data.key_presses == 0:
return BehavioralSignal(
signal_type='no_keyboard',
value=0.0,
timestamp=data.timestamp,
source='keyboard'
)
correction_rate = data.corrections / max(data.key_presses, 1)
correction_score = min(correction_rate * 3, 1.0)
pause_score = min(data.pause_duration / 20, 1.0)
combined = correction_score * 0.6 + pause_score * 0.4
return BehavioralSignal(
signal_type='typing_pattern',
value=combined,
timestamp=data.timestamp,
source='keyboard',
metadata={
'correction_rate': correction_rate,
'pause_duration': data.pause_duration
}
)
def _analyze_focus(self, data: FocusData) -> BehavioralSignal:
"""Analyze focus patterns"""
if data.tab_switches > 0:
return BehavioralSignal(
signal_type='tab_switch',
value=0.7,
timestamp=data.timestamp,
source='focus',
metadata={'switches': data.tab_switches}
)
return BehavioralSignal(
signal_type='focus_level',
value=1.0 if data.is_focused else 0.3,
timestamp=data.timestamp,
source='focus'
)
def calculate_confusion_score(self, signals: List[BehavioralSignal]) -> float:
"""Calculate combined confusion score"""
if not signals:
return 0.0
scores = []
weights = []
mouse_signals = [s for s in signals if s.source == 'mouse']
scroll_signals = [s for s in signals if s.source == 'scroll']
keyboard_signals = [s for s in signals if s.source == 'keyboard']
gesture_signals = [s for s in signals if s.source == 'gesture']
for s in mouse_signals:
if s.signal_type == 'hesitation':
scores.append(s.value)
weights.append(0.2)
for s in scroll_signals:
if s.signal_type == 'scroll_reversal':
scores.append(s.value)
weights.append(0.25)
for s in keyboard_signals:
if s.signal_type == 'typing_pause':
scores.append(s.value)
weights.append(0.15)
for s in gesture_signals:
if s.signal_type in ['confusion', 'cognitive_load']:
scores.append(s.value)
weights.append(0.3)
if not scores:
return 0.0
total_weight = sum(weights) if weights else 1
weighted_sum = sum(s * w for s, w in zip(scores, weights))
return min(weighted_sum / total_weight if total_weight > 0 else 0, 1.0)
def calculate_engagement_score(self) -> float:
"""Calculate overall engagement score"""
if not self.signal_buffer:
return 0.0
recent_signals = list(self.signal_buffer)[-20:]
mouse_count = sum(1 for s in recent_signals if s.source == 'mouse')
keyboard_count = sum(1 for s in recent_signals if s.source == 'keyboard')
scroll_count = sum(1 for s in recent_signals if s.source == 'scroll')
activity_score = (mouse_count + keyboard_count + scroll_count) / 30
focused_signals = [s for s in recent_signals if s.signal_type == 'focus_level' and s.value > 0.5]
focus_score = len(focused_signals) / max(len(recent_signals), 1)
gesture_count = sum(1 for s in recent_signals if s.source == 'gesture')
gesture_score = min(gesture_count / 5, 1.0)
return min((activity_score * 0.4 + focus_score * 0.4 + gesture_score * 0.2), 1.0)
def get_engagement_level(self) -> str:
"""Get engagement level description"""
score = self.calculate_engagement_score()
if score > 0.7:
return "highly_engaged"
elif score > 0.4:
return "moderately_engaged"
elif score > 0.2:
return "low_engagement"
else:
return "disengaged"
def get_session_summary(self) -> Dict:
"""Get session summary"""
duration = (datetime.now() - self.session_start).total_seconds()
return {
'duration_seconds': duration,
'active_time': self.total_active_time,
'break_time': self.total_break_time,
'total_signals': len(self.signal_buffer),
'mouse_events': len(self.mouse_buffer),
'scroll_events': len(self.scroll_buffer),
'keyboard_events': len(self.keyboard_buffer),
'gesture_signals': len(self.gesture_buffer),
'tabs_tracked': len(self.tab_tracking),
'engagement_score': self.calculate_engagement_score(),
'engagement_level': self.get_engagement_level(),
'confusion_score': self.calculate_confusion_score(list(self.signal_buffer)[-20:])
}
def establish_baseline(self, data_points: int = 50):
"""Establish baseline from comfortable learning periods"""
if len(self.signal_buffer) < data_points:
return
recent = list(self.signal_buffer)[-data_points:]
confusion_scores = [s.value for s in recent if 'confusion' in s.signal_type]
self.baseline_metrics = {
'avg_confusion': np.mean(confusion_scores) if confusion_scores else 0.3,
'std_confusion': np.std(confusion_scores) if confusion_scores else 0.2,
'baseline_established': True
}
self.baseline_established = True
def get_behavior_summary(self) -> Dict:
"""Get summary of behavioral analysis"""
recent_signals = list(self.signal_buffer)[-20:]
gesture_signals = [s for s in recent_signals if s.source == 'gesture']
last_gesture = gesture_signals[-1] if gesture_signals else None
return {
'total_signals': len(self.signal_buffer),
'recent_confusion': self.calculate_confusion_score(recent_signals),
'engagement_score': self.calculate_engagement_score(),
'engagement_level': self.get_engagement_level(),
'gesture_recognition_active': len(self.gesture_buffer) > 0,
'last_gesture': last_gesture.metadata if last_gesture else None,
'baseline_established': self.baseline_established,
'session_summary': self.get_session_summary(),
'signal_rates': {
'mouse': len([s for s in recent_signals if s.source == 'mouse']),
'scroll': len([s for s in recent_signals if s.source == 'scroll']),
'keyboard': len([s for s in recent_signals if s.source == 'keyboard']),
'gesture': len([s for s in recent_signals if s.source == 'gesture'])
}
}