| """ |
| Behavioral Agent - Updated without camera eye tracking |
| |
| Tracks behavioral signals through: |
| - Mouse movements (hesitation, click patterns) |
| - Scroll behavior (reversals, speed) |
| - Keyboard patterns (typing speed, corrections) |
| - Time patterns (learning duration, break frequency) |
| - Hand gestures (trained by user) |
| - Page focus/blur events |
| - Selection patterns |
| """ |
|
|
| import time |
| from typing import Dict, List, Any, Optional |
| from dataclasses import dataclass, field |
| from datetime import datetime |
| from collections import deque |
| import numpy as np |
|
|
|
|
| @dataclass |
| class MouseData: |
| """Mouse movement data""" |
| x: float |
| y: float |
| velocity: float |
| acceleration: float |
| click_count: int |
| selection_count: int |
| right_click: bool |
| timestamp: datetime |
|
|
|
|
| @dataclass |
| class ScrollData: |
| """Scroll behavior data""" |
| scroll_up_count: int |
| scroll_down_count: int |
| reversal_count: int |
| avg_scroll_speed: float |
| max_scroll_speed: float |
| direction_changes: int |
| timestamp: datetime |
|
|
|
|
| @dataclass |
| class KeyboardData: |
| """Keyboard activity data""" |
| key_presses: int |
| backspaces: int |
| typing_speed: float |
| corrections: int |
| pause_duration: float |
| timestamp: datetime |
|
|
|
|
| @dataclass |
| class FocusData: |
| """Tab/window focus data""" |
| is_focused: bool |
| visible_duration: float |
| hidden_duration: float |
| tab_switches: int |
| timestamp: datetime |
|
|
|
|
| @dataclass |
| class BehavioralSignal: |
| """Aggregated behavioral signal""" |
| signal_type: str |
| value: float |
| timestamp: datetime |
| source: str |
| metadata: Dict = field(default_factory=dict) |
|
|
|
|
| @dataclass |
| class TabActivity: |
| """Tab activity tracking""" |
| tab_id: str |
| url: str |
| title: str |
| active_time: float = 0 |
| scroll_depth: float = 0 |
| clicks: int = 0 |
| keystrokes: int = 0 |
| focus_count: int = 0 |
|
|
|
|
| class BehavioralAgent: |
| """ |
| Agent that tracks behavioral signals without camera access. |
| |
| Signal Sources: |
| 1. Mouse - movements, clicks, selections |
| 2. Scroll - patterns, reversals, speed |
| 3. Keyboard - typing, corrections, pauses |
| 4. Focus - tab switches, visibility |
| 5. Hand Gestures - trained by user |
| """ |
| |
| def __init__(self, user_id: str, config: Optional[Dict] = None): |
| self.user_id = user_id |
| self.config = config or {} |
| |
| self.signal_buffer = deque(maxlen=100) |
| self.mouse_buffer = deque(maxlen=50) |
| self.scroll_buffer = deque(maxlen=50) |
| self.keyboard_buffer = deque(maxlen=50) |
| self.focus_buffer = deque(maxlen=50) |
| self.gesture_buffer = deque(maxlen=20) |
| |
| self.baseline_established = False |
| self.baseline_metrics = {} |
| |
| self.tab_tracking: Dict[str, TabActivity] = {} |
| self.current_tab: Optional[str] = None |
| |
| self.session_start = datetime.now() |
| self.total_active_time = 0 |
| self.total_break_time = 0 |
| |
| self.last_mouse_move = None |
| self.last_scroll_time = None |
| self.last_keypress = None |
| self.scroll_direction = 1 |
| |
| def add_mouse_data(self, data: Dict) -> MouseData: |
| """Add mouse movement data""" |
| mouse_data = MouseData( |
| x=data.get('x', 0), |
| y=data.get('y', 0), |
| velocity=data.get('velocity', 0), |
| acceleration=data.get('acceleration', 0), |
| click_count=data.get('click_count', 0), |
| selection_count=data.get('selection_count', 0), |
| right_click=data.get('right_click', False), |
| timestamp=datetime.now() |
| ) |
| |
| self.mouse_buffer.append(mouse_data) |
| |
| if self.last_mouse_move: |
| dt = (mouse_data.timestamp - self.last_mouse_move).total_seconds() |
| if dt > 2: |
| signal = BehavioralSignal( |
| signal_type='hesitation', |
| value=min(dt / 10, 1.0), |
| timestamp=mouse_data.timestamp, |
| source='mouse', |
| metadata={'duration': dt} |
| ) |
| self.signal_buffer.append(signal) |
| |
| self.last_mouse_move = mouse_data.timestamp |
| return mouse_data |
| |
| def add_scroll_data(self, data: Dict) -> ScrollData: |
| """Add scroll behavior data""" |
| direction = data.get('direction', 'down') |
| |
| if self.last_scroll_time: |
| dt = (datetime.now() - self.last_scroll_time).total_seconds() |
| if dt > 0.5 and direction != self.scroll_direction: |
| reversal = BehavioralSignal( |
| signal_type='scroll_reversal', |
| value=0.5, |
| timestamp=datetime.now(), |
| source='scroll' |
| ) |
| self.signal_buffer.append(reversal) |
| |
| self.scroll_direction = 1 if direction == 'down' else -1 |
| self.last_scroll_time = datetime.now() |
| |
| scroll_data = ScrollData( |
| scroll_up_count=data.get('scroll_up_count', 0), |
| scroll_down_count=data.get('scroll_down_count', 0), |
| reversal_count=data.get('reversal_count', 0), |
| avg_scroll_speed=data.get('avg_scroll_speed', 0), |
| max_scroll_speed=data.get('max_scroll_speed', 0), |
| direction_changes=data.get('direction_changes', 0), |
| timestamp=datetime.now() |
| ) |
| |
| self.scroll_buffer.append(scroll_data) |
| return scroll_data |
| |
| def add_keyboard_data(self, data: Dict) -> KeyboardData: |
| """Add keyboard activity data""" |
| keyboard_data = KeyboardData( |
| key_presses=data.get('key_presses', 0), |
| backspaces=data.get('backspaces', 0), |
| typing_speed=data.get('typing_speed', 0), |
| corrections=data.get('corrections', 0), |
| pause_duration=data.get('pause_duration', 0), |
| timestamp=datetime.now() |
| ) |
| |
| self.keyboard_buffer.append(keyboard_data) |
| |
| if keyboard_data.pause_duration > 5: |
| signal = BehavioralSignal( |
| signal_type='typing_pause', |
| value=min(keyboard_data.pause_duration / 30, 1.0), |
| timestamp=keyboard_data.timestamp, |
| source='keyboard', |
| metadata={'pause_duration': keyboard_data.pause_duration} |
| ) |
| self.signal_buffer.append(signal) |
| |
| return keyboard_data |
| |
| def add_focus_data(self, data: Dict) -> FocusData: |
| """Add tab/window focus data""" |
| focus_data = FocusData( |
| is_focused=data.get('is_focused', True), |
| visible_duration=data.get('visible_duration', 0), |
| hidden_duration=data.get('hidden_duration', 0), |
| tab_switches=data.get('tab_switches', 0), |
| timestamp=datetime.now() |
| ) |
| |
| self.focus_buffer.append(focus_data) |
| |
| if not focus_data.is_focused: |
| signal = BehavioralSignal( |
| signal_type='unfocused', |
| value=0.8, |
| timestamp=focus_data.timestamp, |
| source='focus' |
| ) |
| self.signal_buffer.append(signal) |
| |
| return focus_data |
| |
| def add_tab_data(self, data: Dict): |
| """Track tab activity""" |
| tab_id = data.get('tab_id') |
| if not tab_id: |
| return |
| |
| if tab_id not in self.tab_tracking: |
| self.tab_tracking[tab_id] = TabActivity( |
| tab_id=tab_id, |
| url=data.get('url', ''), |
| title=data.get('title', '') |
| ) |
| |
| tab = self.tab_tracking[tab_id] |
| |
| if 'scroll_depth' in data: |
| tab.scroll_depth = data['scroll_depth'] |
| if 'clicks' in data: |
| tab.clicks += data['clicks'] |
| if 'keystrokes' in data: |
| tab.keystrokes += data['keystrokes'] |
| if 'focus' in data and data['focus']: |
| tab.focus_count += 1 |
| |
| def add_gesture_signal(self, gesture_signal: Dict): |
| """Add hand gesture signal""" |
| signal = BehavioralSignal( |
| signal_type=gesture_signal.get('signal_type', 'unknown'), |
| value=gesture_signal.get('confidence', 0.5), |
| timestamp=datetime.fromisoformat(gesture_signal.get('timestamp', datetime.now().isoformat())), |
| source='gesture', |
| metadata={ |
| 'gesture_name': gesture_signal.get('gesture_name', ''), |
| 'raw_confidence': gesture_signal.get('raw_confidence', 0), |
| 'description': gesture_signal.get('description', '') |
| } |
| ) |
| |
| self.gesture_buffer.append(signal) |
| self.signal_buffer.append(signal) |
| |
| def process_signals(self, data: Dict) -> List[BehavioralSignal]: |
| """Process all incoming behavioral signals""" |
| signals = [] |
| |
| if 'mouse' in data: |
| mouse_data = self.add_mouse_data(data['mouse']) |
| signal = self._analyze_mouse_pattern(mouse_data) |
| signals.append(signal) |
| |
| if 'scroll' in data: |
| scroll_data = self.add_scroll_data(data['scroll']) |
| signal = self._analyze_scroll_behavior(scroll_data) |
| signals.append(signal) |
| |
| if 'keyboard' in data: |
| keyboard_data = self.add_keyboard_data(data['keyboard']) |
| signal = self._analyze_keyboard_pattern(keyboard_data) |
| signals.append(signal) |
| |
| if 'focus' in data: |
| focus_data = self.add_focus_data(data['focus']) |
| signal = self._analyze_focus(focus_data) |
| signals.append(signal) |
| |
| if 'gesture' in data: |
| self.add_gesture_signal(data['gesture']) |
| |
| if 'tab' in data: |
| self.add_tab_data(data['tab']) |
| |
| return signals |
| |
| def _analyze_mouse_pattern(self, data: MouseData) -> BehavioralSignal: |
| """Analyze mouse patterns""" |
| hesitation_threshold = 50 |
| velocity_threshold = 100 |
| |
| hesitation_score = min(data.acceleration / hesitation_threshold, 1.0) |
| velocity_score = min(data.velocity / velocity_threshold, 1.0) |
| |
| combined = (hesitation_score * 0.6 + velocity_score * 0.4) |
| |
| if data.right_click: |
| combined = max(combined, 0.6) |
| |
| return BehavioralSignal( |
| signal_type='mouse_activity', |
| value=combined, |
| timestamp=data.timestamp, |
| source='mouse', |
| metadata={ |
| 'hesitation': hesitation_score, |
| 'velocity': velocity_score, |
| 'clicks': data.click_count |
| } |
| ) |
| |
| def _analyze_scroll_behavior(self, data: ScrollData) -> BehavioralSignal: |
| """Analyze scroll behavior""" |
| if data.scroll_up_count + data.scroll_down_count == 0: |
| return BehavioralSignal( |
| signal_type='scroll_inactive', |
| value=0.0, |
| timestamp=data.timestamp, |
| source='scroll' |
| ) |
| |
| reversal_ratio = data.reversal_count / max( |
| data.scroll_up_count + data.scroll_down_count, 1 |
| ) |
| |
| reversal_score = min(reversal_ratio * 2, 1.0) |
| |
| speed_variation = 0 |
| if data.max_scroll_speed > 0: |
| speed_variation = abs(data.avg_scroll_speed - data.max_scroll_speed) / data.max_scroll_speed |
| |
| combined = reversal_score * 0.7 + speed_variation * 0.3 |
| |
| return BehavioralSignal( |
| signal_type='scroll_pattern', |
| value=combined, |
| timestamp=data.timestamp, |
| source='scroll', |
| metadata={ |
| 'reversal_ratio': reversal_ratio, |
| 'speed_variation': speed_variation |
| } |
| ) |
| |
| def _analyze_keyboard_pattern(self, data: KeyboardData) -> BehavioralSignal: |
| """Analyze keyboard patterns""" |
| if data.key_presses == 0: |
| return BehavioralSignal( |
| signal_type='no_keyboard', |
| value=0.0, |
| timestamp=data.timestamp, |
| source='keyboard' |
| ) |
| |
| correction_rate = data.corrections / max(data.key_presses, 1) |
| |
| correction_score = min(correction_rate * 3, 1.0) |
| |
| pause_score = min(data.pause_duration / 20, 1.0) |
| |
| combined = correction_score * 0.6 + pause_score * 0.4 |
| |
| return BehavioralSignal( |
| signal_type='typing_pattern', |
| value=combined, |
| timestamp=data.timestamp, |
| source='keyboard', |
| metadata={ |
| 'correction_rate': correction_rate, |
| 'pause_duration': data.pause_duration |
| } |
| ) |
| |
| def _analyze_focus(self, data: FocusData) -> BehavioralSignal: |
| """Analyze focus patterns""" |
| if data.tab_switches > 0: |
| return BehavioralSignal( |
| signal_type='tab_switch', |
| value=0.7, |
| timestamp=data.timestamp, |
| source='focus', |
| metadata={'switches': data.tab_switches} |
| ) |
| |
| return BehavioralSignal( |
| signal_type='focus_level', |
| value=1.0 if data.is_focused else 0.3, |
| timestamp=data.timestamp, |
| source='focus' |
| ) |
| |
| def calculate_confusion_score(self, signals: List[BehavioralSignal]) -> float: |
| """Calculate combined confusion score""" |
| if not signals: |
| return 0.0 |
| |
| scores = [] |
| weights = [] |
| |
| mouse_signals = [s for s in signals if s.source == 'mouse'] |
| scroll_signals = [s for s in signals if s.source == 'scroll'] |
| keyboard_signals = [s for s in signals if s.source == 'keyboard'] |
| gesture_signals = [s for s in signals if s.source == 'gesture'] |
| |
| for s in mouse_signals: |
| if s.signal_type == 'hesitation': |
| scores.append(s.value) |
| weights.append(0.2) |
| |
| for s in scroll_signals: |
| if s.signal_type == 'scroll_reversal': |
| scores.append(s.value) |
| weights.append(0.25) |
| |
| for s in keyboard_signals: |
| if s.signal_type == 'typing_pause': |
| scores.append(s.value) |
| weights.append(0.15) |
| |
| for s in gesture_signals: |
| if s.signal_type in ['confusion', 'cognitive_load']: |
| scores.append(s.value) |
| weights.append(0.3) |
| |
| if not scores: |
| return 0.0 |
| |
| total_weight = sum(weights) if weights else 1 |
| weighted_sum = sum(s * w for s, w in zip(scores, weights)) |
| |
| return min(weighted_sum / total_weight if total_weight > 0 else 0, 1.0) |
| |
| def calculate_engagement_score(self) -> float: |
| """Calculate overall engagement score""" |
| if not self.signal_buffer: |
| return 0.0 |
| |
| recent_signals = list(self.signal_buffer)[-20:] |
| |
| mouse_count = sum(1 for s in recent_signals if s.source == 'mouse') |
| keyboard_count = sum(1 for s in recent_signals if s.source == 'keyboard') |
| scroll_count = sum(1 for s in recent_signals if s.source == 'scroll') |
| |
| activity_score = (mouse_count + keyboard_count + scroll_count) / 30 |
| |
| focused_signals = [s for s in recent_signals if s.signal_type == 'focus_level' and s.value > 0.5] |
| focus_score = len(focused_signals) / max(len(recent_signals), 1) |
| |
| gesture_count = sum(1 for s in recent_signals if s.source == 'gesture') |
| gesture_score = min(gesture_count / 5, 1.0) |
| |
| return min((activity_score * 0.4 + focus_score * 0.4 + gesture_score * 0.2), 1.0) |
| |
| def get_engagement_level(self) -> str: |
| """Get engagement level description""" |
| score = self.calculate_engagement_score() |
| |
| if score > 0.7: |
| return "highly_engaged" |
| elif score > 0.4: |
| return "moderately_engaged" |
| elif score > 0.2: |
| return "low_engagement" |
| else: |
| return "disengaged" |
| |
| def get_session_summary(self) -> Dict: |
| """Get session summary""" |
| duration = (datetime.now() - self.session_start).total_seconds() |
| |
| return { |
| 'duration_seconds': duration, |
| 'active_time': self.total_active_time, |
| 'break_time': self.total_break_time, |
| 'total_signals': len(self.signal_buffer), |
| 'mouse_events': len(self.mouse_buffer), |
| 'scroll_events': len(self.scroll_buffer), |
| 'keyboard_events': len(self.keyboard_buffer), |
| 'gesture_signals': len(self.gesture_buffer), |
| 'tabs_tracked': len(self.tab_tracking), |
| 'engagement_score': self.calculate_engagement_score(), |
| 'engagement_level': self.get_engagement_level(), |
| 'confusion_score': self.calculate_confusion_score(list(self.signal_buffer)[-20:]) |
| } |
| |
| def establish_baseline(self, data_points: int = 50): |
| """Establish baseline from comfortable learning periods""" |
| if len(self.signal_buffer) < data_points: |
| return |
| |
| recent = list(self.signal_buffer)[-data_points:] |
| |
| confusion_scores = [s.value for s in recent if 'confusion' in s.signal_type] |
| |
| self.baseline_metrics = { |
| 'avg_confusion': np.mean(confusion_scores) if confusion_scores else 0.3, |
| 'std_confusion': np.std(confusion_scores) if confusion_scores else 0.2, |
| 'baseline_established': True |
| } |
| |
| self.baseline_established = True |
| |
| def get_behavior_summary(self) -> Dict: |
| """Get summary of behavioral analysis""" |
| recent_signals = list(self.signal_buffer)[-20:] |
| |
| gesture_signals = [s for s in recent_signals if s.source == 'gesture'] |
| last_gesture = gesture_signals[-1] if gesture_signals else None |
| |
| return { |
| 'total_signals': len(self.signal_buffer), |
| 'recent_confusion': self.calculate_confusion_score(recent_signals), |
| 'engagement_score': self.calculate_engagement_score(), |
| 'engagement_level': self.get_engagement_level(), |
| 'gesture_recognition_active': len(self.gesture_buffer) > 0, |
| 'last_gesture': last_gesture.metadata if last_gesture else None, |
| 'baseline_established': self.baseline_established, |
| 'session_summary': self.get_session_summary(), |
| 'signal_rates': { |
| 'mouse': len([s for s in recent_signals if s.source == 'mouse']), |
| 'scroll': len([s for s in recent_signals if s.source == 'scroll']), |
| 'keyboard': len([s for s in recent_signals if s.source == 'keyboard']), |
| 'gesture': len([s for s in recent_signals if s.source == 'gesture']) |
| } |
| } |
|
|