DocUA commited on
Commit
74541bd
Β·
1 Parent(s): 6683b63

feat: Add conversation verification feature with UI and backend support

Browse files

- Implemented a new verification section in the Gradio interface to allow users to verify conversation exchanges.
- Created core models and manager for handling verification sessions, including feedback and progress tracking.
- Developed a CSV exporter for verification results, enabling users to export session data.
- Built a comprehensive Gradio UI for reviewing and verifying AI classifier decisions, including navigation and statistics display.
- Added functionality to mark exchanges as correct or incorrect, with options for providing feedback and reasons for corrections.
- Ensured session data is saved and loaded correctly, maintaining state across user interactions.

.gitignore CHANGED
@@ -65,6 +65,8 @@ flagged/
65
  # Hypothesis testing
66
  .hypothesis/
67
  .verification_data/
 
 
68
 
69
  # Logs
70
  *.log
 
65
  # Hypothesis testing
66
  .hypothesis/
67
  .verification_data/
68
+ verification_sessions/
69
+ verification_exports/
70
 
71
  # Logs
72
  *.log
src/core/conversation_verification.py ADDED
@@ -0,0 +1,292 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """
3
+ Conversation Verification System - Core Models and Manager.
4
+
5
+ Provides data models and management functionality for verifying AI classifier
6
+ decisions made during patient conversations.
7
+ """
8
+
9
+ import json
10
+ import os
11
+ import uuid
12
+ from datetime import datetime
13
+ from typing import Dict, List, Any, Optional, Tuple
14
+ from dataclasses import dataclass, asdict, field
15
+
16
+ from src.core.conversation_logger import ConversationLogger, ConversationEntry
17
+
18
+
19
+ @dataclass
20
+ class VerificationFeedback:
21
+ """Feedback provided by verifier for a conversation exchange."""
22
+ exchange_id: str
23
+ is_correct: bool
24
+ correct_classification: Optional[str] = None # Required if is_correct=False
25
+ correction_reason: Optional[str] = None
26
+ notes: Optional[str] = None
27
+
28
+
29
+ @dataclass
30
+ class VerificationRecord:
31
+ """Complete verification record for a single conversation exchange."""
32
+ exchange_id: str
33
+ exchange_number: int
34
+ timestamp: datetime
35
+ user_message: str
36
+ assistant_response: str
37
+ original_classification: str # GREEN/YELLOW/RED
38
+ original_confidence: float
39
+ original_indicators: List[str]
40
+ original_reasoning: str
41
+ is_correct: Optional[bool] = None
42
+ correct_classification: Optional[str] = None
43
+ correction_reason: Optional[str] = None
44
+ verifier_notes: Optional[str] = None
45
+ verification_timestamp: Optional[datetime] = None
46
+
47
+ @classmethod
48
+ def from_conversation_entry(cls, entry: ConversationEntry, exchange_number: int) -> 'VerificationRecord':
49
+ """Create VerificationRecord from ConversationEntry."""
50
+ return cls(
51
+ exchange_id=f"{entry.session_id}_{entry.message_index}",
52
+ exchange_number=exchange_number,
53
+ timestamp=datetime.fromisoformat(entry.timestamp),
54
+ user_message=entry.user_message,
55
+ assistant_response=entry.assistant_response,
56
+ original_classification=entry.spiritual_classification,
57
+ original_confidence=entry.classification_confidence,
58
+ original_indicators=entry.classification_indicators.copy(),
59
+ original_reasoning=entry.classification_reasoning
60
+ )
61
+
62
+ def apply_feedback(self, feedback: VerificationFeedback) -> None:
63
+ """Apply verification feedback to this record."""
64
+ self.is_correct = feedback.is_correct
65
+ self.correct_classification = feedback.correct_classification
66
+ self.correction_reason = feedback.correction_reason
67
+ self.verifier_notes = feedback.notes
68
+ self.verification_timestamp = datetime.now()
69
+
70
+
71
+ @dataclass
72
+ class VerificationProgress:
73
+ """Progress tracking for verification session."""
74
+ total_exchanges: int
75
+ verified_exchanges: int
76
+ accuracy_overall: float = 0.0
77
+ accuracy_by_type: Dict[str, float] = field(default_factory=dict)
78
+ common_errors: List[Tuple[str, str, int]] = field(default_factory=list)
79
+
80
+ def calculate_progress_percentage(self) -> float:
81
+ """Calculate verification progress as percentage."""
82
+ if self.total_exchanges == 0:
83
+ return 0.0
84
+ return (self.verified_exchanges / self.total_exchanges) * 100
85
+
86
+ def is_complete(self) -> bool:
87
+ """Check if verification is complete."""
88
+ return self.verified_exchanges == self.total_exchanges
89
+
90
+
91
+ @dataclass
92
+ class VerificationSession:
93
+ """Complete verification session for a conversation."""
94
+ session_id: str
95
+ conversation_session_id: str # Links to ConversationLogger session
96
+ patient_name: str
97
+ verifier_name: str
98
+ start_time: datetime
99
+ end_time: Optional[datetime] = None
100
+ total_exchanges: int = 0
101
+ verified_exchanges: int = 0
102
+ verification_records: List[VerificationRecord] = field(default_factory=list)
103
+ is_complete: bool = False
104
+
105
+ def get_progress(self) -> VerificationProgress:
106
+ """Get current verification progress."""
107
+ # Calculate overall accuracy
108
+ verified_records = [r for r in self.verification_records if r.is_correct is not None]
109
+ correct_count = sum(1 for r in verified_records if r.is_correct)
110
+ accuracy_overall = (correct_count / len(verified_records)) if verified_records else 0.0
111
+
112
+ # Calculate accuracy by classification type
113
+ accuracy_by_type = {}
114
+ for classification in ['GREEN', 'YELLOW', 'RED']:
115
+ type_records = [r for r in verified_records if r.original_classification == classification]
116
+ if type_records:
117
+ type_correct = sum(1 for r in type_records if r.is_correct)
118
+ accuracy_by_type[classification] = type_correct / len(type_records)
119
+ else:
120
+ accuracy_by_type[classification] = 0.0
121
+
122
+ # Find common errors
123
+ error_patterns = {}
124
+ for record in verified_records:
125
+ if not record.is_correct and record.correct_classification:
126
+ error_key = (record.original_classification, record.correct_classification)
127
+ error_patterns[error_key] = error_patterns.get(error_key, 0) + 1
128
+
129
+ common_errors = [(from_class, to_class, count)
130
+ for (from_class, to_class), count in
131
+ sorted(error_patterns.items(), key=lambda x: x[1], reverse=True)[:5]]
132
+
133
+ return VerificationProgress(
134
+ total_exchanges=self.total_exchanges,
135
+ verified_exchanges=len(verified_records),
136
+ accuracy_overall=accuracy_overall,
137
+ accuracy_by_type=accuracy_by_type,
138
+ common_errors=common_errors
139
+ )
140
+
141
+ def add_verification_record(self, record: VerificationRecord) -> None:
142
+ """Add verification record to session."""
143
+ self.verification_records.append(record)
144
+
145
+ def apply_feedback(self, exchange_id: str, feedback: VerificationFeedback) -> bool:
146
+ """Apply feedback to specific exchange."""
147
+ for record in self.verification_records:
148
+ if record.exchange_id == exchange_id:
149
+ record.apply_feedback(feedback)
150
+ self.verified_exchanges = len([r for r in self.verification_records if r.is_correct is not None])
151
+
152
+ # Check if session is complete
153
+ if self.verified_exchanges == self.total_exchanges:
154
+ self.is_complete = True
155
+ self.end_time = datetime.now()
156
+
157
+ return True
158
+ return False
159
+
160
+ def get_unverified_records(self) -> List[VerificationRecord]:
161
+ """Get list of unverified records."""
162
+ return [r for r in self.verification_records if r.is_correct is None]
163
+
164
+ def get_next_unverified_record(self) -> Optional[VerificationRecord]:
165
+ """Get next unverified record."""
166
+ unverified = self.get_unverified_records()
167
+ return unverified[0] if unverified else None
168
+
169
+
170
+ class ConversationVerificationManager:
171
+ """Manager for conversation verification sessions."""
172
+
173
+ def __init__(self, storage_dir: str = "verification_sessions"):
174
+ """Initialize verification manager."""
175
+ from src.core.verification_store import JSONVerificationStore
176
+ self.store = JSONVerificationStore(storage_dir)
177
+
178
+ def create_verification_session(
179
+ self,
180
+ conversation_logger: ConversationLogger,
181
+ verifier_name: str = "Medical Professional"
182
+ ) -> VerificationSession:
183
+ """
184
+ Create new verification session from conversation logger.
185
+
186
+ Args:
187
+ conversation_logger: Source conversation to verify
188
+ verifier_name: Name of person doing verification
189
+
190
+ Returns:
191
+ New VerificationSession ready for verification
192
+ """
193
+ session_id = f"verification_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{str(uuid.uuid4())[:8]}"
194
+
195
+ # Create verification session
196
+ session = VerificationSession(
197
+ session_id=session_id,
198
+ conversation_session_id=conversation_logger.session_id,
199
+ patient_name=conversation_logger.patient_name,
200
+ verifier_name=verifier_name,
201
+ start_time=datetime.now(),
202
+ total_exchanges=len(conversation_logger.entries)
203
+ )
204
+
205
+ # Convert conversation entries to verification records
206
+ for i, entry in enumerate(conversation_logger.entries, 1):
207
+ record = VerificationRecord.from_conversation_entry(entry, i)
208
+ session.add_verification_record(record)
209
+
210
+ # Save initial session
211
+ self.store.save_session(session)
212
+
213
+ return session
214
+
215
+ def get_verification_progress(self, session_id: str) -> Optional[VerificationProgress]:
216
+ """Get verification progress for session."""
217
+ session = self.store.load_session(session_id)
218
+ return session.get_progress() if session else None
219
+
220
+ def submit_exchange_verification(
221
+ self,
222
+ session_id: str,
223
+ exchange_id: str,
224
+ feedback: VerificationFeedback
225
+ ) -> bool:
226
+ """
227
+ Submit verification feedback for an exchange.
228
+
229
+ Args:
230
+ session_id: Verification session ID
231
+ exchange_id: Exchange being verified
232
+ feedback: Verification feedback
233
+
234
+ Returns:
235
+ True if feedback was applied successfully
236
+ """
237
+ session = self.store.load_session(session_id)
238
+ if not session:
239
+ return False
240
+
241
+ # Validate feedback
242
+ if not feedback.is_correct and not feedback.correct_classification:
243
+ raise ValueError("correct_classification required when is_correct=False")
244
+
245
+ if (not feedback.is_correct and
246
+ feedback.correct_classification and
247
+ feedback.correct_classification not in ['GREEN', 'YELLOW', 'RED']):
248
+ raise ValueError("correct_classification must be GREEN, YELLOW, or RED")
249
+
250
+ # Apply feedback
251
+ success = session.apply_feedback(exchange_id, feedback)
252
+ if success:
253
+ self.store.save_session(session)
254
+
255
+ return success
256
+
257
+ def get_session_statistics(self, session_id: str) -> Optional[Dict[str, Any]]:
258
+ """Get detailed statistics for verification session."""
259
+ session = self.store.load_session(session_id)
260
+ if not session:
261
+ return None
262
+
263
+ progress = session.get_progress()
264
+
265
+ return {
266
+ "session_id": session.session_id,
267
+ "patient_name": session.patient_name,
268
+ "verifier_name": session.verifier_name,
269
+ "start_time": session.start_time.isoformat(),
270
+ "end_time": session.end_time.isoformat() if session.end_time else None,
271
+ "is_complete": session.is_complete,
272
+ "progress": asdict(progress),
273
+ "total_exchanges": session.total_exchanges,
274
+ "verified_exchanges": session.verified_exchanges
275
+ }
276
+
277
+ def load_session(self, session_id: str) -> Optional[VerificationSession]:
278
+ """Load verification session by ID."""
279
+ return self.store.load_session(session_id)
280
+
281
+ def save_session(self, session: VerificationSession) -> None:
282
+ """Save verification session."""
283
+ self.store.save_session(session)
284
+
285
+ def list_sessions(self) -> List[Dict[str, Any]]:
286
+ """List all verification sessions."""
287
+ return self.store.list_sessions()
288
+
289
+ def get_incomplete_sessions(self) -> List[Dict[str, Any]]:
290
+ """Get incomplete verification sessions."""
291
+ return self.store.get_incomplete_sessions()
292
+
src/core/verification_exporter.py ADDED
@@ -0,0 +1,152 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """
3
+ Verification Results Exporter.
4
+
5
+ Handles exporting verification session results to CSV format.
6
+ """
7
+
8
+ import csv
9
+ import os
10
+ from datetime import datetime
11
+ from typing import List, Dict, Any
12
+ from src.core.conversation_verification import VerificationSession, VerificationRecord
13
+
14
+
15
+ class VerificationExporter:
16
+ """Exporter for verification session results."""
17
+
18
+ def __init__(self, export_dir: str = "verification_exports"):
19
+ """Initialize exporter."""
20
+ self.export_dir = export_dir
21
+ os.makedirs(export_dir, exist_ok=True)
22
+
23
+ def export_session_to_csv(self, session: VerificationSession) -> str:
24
+ """
25
+ Export verification session to CSV format.
26
+
27
+ Args:
28
+ session: VerificationSession to export
29
+
30
+ Returns:
31
+ Path to exported CSV file
32
+ """
33
+ filename = self._create_export_filename(session)
34
+ filepath = os.path.join(self.export_dir, filename)
35
+
36
+ try:
37
+ with open(filepath, 'w', newline='', encoding='utf-8') as csvfile:
38
+ fieldnames = [
39
+ 'exchange_number',
40
+ 'timestamp',
41
+ 'user_message',
42
+ 'assistant_response',
43
+ 'classifier_decision',
44
+ 'classifier_confidence',
45
+ 'indicators',
46
+ 'reasoning',
47
+ 'is_correct',
48
+ 'correct_classification',
49
+ 'correction_reason',
50
+ 'verifier_notes',
51
+ 'verification_timestamp'
52
+ ]
53
+
54
+ writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
55
+ writer.writeheader()
56
+
57
+ # Write session metadata as comment rows
58
+ writer.writerow({
59
+ 'exchange_number': '# SESSION METADATA',
60
+ 'timestamp': f'Session ID: {session.session_id}',
61
+ 'user_message': f'Patient: {session.patient_name}',
62
+ 'assistant_response': f'Verifier: {session.verifier_name}',
63
+ 'classifier_decision': f'Started: {session.start_time.isoformat()}',
64
+ 'classifier_confidence': f'Completed: {session.end_time.isoformat() if session.end_time else "In Progress"}',
65
+ 'indicators': f'Total Exchanges: {session.total_exchanges}',
66
+ 'reasoning': f'Verified: {session.verified_exchanges}',
67
+ 'is_correct': f'Complete: {session.is_complete}',
68
+ 'correct_classification': '',
69
+ 'correction_reason': '',
70
+ 'verifier_notes': '',
71
+ 'verification_timestamp': ''
72
+ })
73
+
74
+ # Add empty row for separation
75
+ writer.writerow({field: '' for field in fieldnames})
76
+
77
+ # Write verification records
78
+ for record in session.verification_records:
79
+ writer.writerow({
80
+ 'exchange_number': record.exchange_number,
81
+ 'timestamp': record.timestamp.isoformat(),
82
+ 'user_message': record.user_message,
83
+ 'assistant_response': record.assistant_response,
84
+ 'classifier_decision': record.original_classification,
85
+ 'classifier_confidence': record.original_confidence,
86
+ 'indicators': '; '.join(record.original_indicators),
87
+ 'reasoning': record.original_reasoning,
88
+ 'is_correct': record.is_correct,
89
+ 'correct_classification': record.correct_classification or '',
90
+ 'correction_reason': record.correction_reason or '',
91
+ 'verifier_notes': record.verifier_notes or '',
92
+ 'verification_timestamp': record.verification_timestamp.isoformat() if record.verification_timestamp else ''
93
+ })
94
+
95
+ return filepath
96
+
97
+ except Exception as e:
98
+ raise Exception(f"Failed to export CSV: {str(e)}")
99
+
100
+ def _create_export_filename(self, session: VerificationSession) -> str:
101
+ """Create descriptive filename for export."""
102
+ timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
103
+ patient_safe = session.patient_name.replace(' ', '_').replace('/', '_')
104
+ return f"conversation_verification_{patient_safe}_{timestamp}.csv"
105
+
106
+ def generate_summary_report(self, session: VerificationSession) -> Dict[str, Any]:
107
+ """Generate summary report for verification session."""
108
+ progress = session.get_progress()
109
+
110
+ # Calculate detailed statistics
111
+ verified_records = [r for r in session.verification_records if r.is_correct is not None]
112
+ correct_records = [r for r in verified_records if r.is_correct]
113
+ incorrect_records = [r for r in verified_records if not r.is_correct]
114
+
115
+ # Error analysis
116
+ error_patterns = {}
117
+ for record in incorrect_records:
118
+ if record.correct_classification:
119
+ error_key = f"{record.original_classification} β†’ {record.correct_classification}"
120
+ error_patterns[error_key] = error_patterns.get(error_key, 0) + 1
121
+
122
+ # Confidence analysis
123
+ confidence_stats = {
124
+ 'avg_confidence': sum(r.original_confidence for r in verified_records) / len(verified_records) if verified_records else 0,
125
+ 'correct_avg_confidence': sum(r.original_confidence for r in correct_records) / len(correct_records) if correct_records else 0,
126
+ 'incorrect_avg_confidence': sum(r.original_confidence for r in incorrect_records) / len(incorrect_records) if incorrect_records else 0
127
+ }
128
+
129
+ return {
130
+ 'session_info': {
131
+ 'session_id': session.session_id,
132
+ 'patient_name': session.patient_name,
133
+ 'verifier_name': session.verifier_name,
134
+ 'start_time': session.start_time.isoformat(),
135
+ 'end_time': session.end_time.isoformat() if session.end_time else None,
136
+ 'duration_minutes': (session.end_time - session.start_time).total_seconds() / 60 if session.end_time else None
137
+ },
138
+ 'verification_stats': {
139
+ 'total_exchanges': session.total_exchanges,
140
+ 'verified_exchanges': len(verified_records),
141
+ 'completion_rate': len(verified_records) / session.total_exchanges if session.total_exchanges > 0 else 0,
142
+ 'overall_accuracy': progress.accuracy_overall,
143
+ 'accuracy_by_type': progress.accuracy_by_type
144
+ },
145
+ 'error_analysis': {
146
+ 'total_errors': len(incorrect_records),
147
+ 'error_rate': len(incorrect_records) / len(verified_records) if verified_records else 0,
148
+ 'error_patterns': error_patterns,
149
+ 'common_errors': progress.common_errors
150
+ },
151
+ 'confidence_analysis': confidence_stats
152
+ }
src/core/verification_store.py CHANGED
@@ -1,1249 +1,350 @@
1
- # verification_store.py
2
  """
3
- Verification data storage layer.
4
 
5
- Provides interface and JSON-based implementation for persisting verification data.
6
- Enhanced to support multi-mode verification sessions with comprehensive export capabilities.
7
  """
8
 
9
  import json
10
  import os
11
- import csv
12
- import io
13
- import logging
14
- from abc import ABC, abstractmethod
15
- from typing import Dict, List, Optional, Any, Union, Tuple
16
  from datetime import datetime
17
- from pathlib import Path
 
18
 
19
- from src.core.verification_models import (
20
- VerificationSession,
21
- VerificationRecord,
22
- TestDataset,
23
- TestMessage,
24
- EnhancedVerificationSession,
25
- TestCaseEdit,
26
- FileUploadResult,
27
- )
28
- from src.core.enhanced_error_handler import EnhancedErrorHandler, ErrorCategory
29
- from src.core.error_handling_utils import ErrorHandlingDecorator
30
- from src.core.data_validation_service import DataValidationService, IntegrityChecksum
31
 
32
 
33
- class VerificationDataStore(ABC):
34
- """Abstract interface for verification data storage."""
35
 
36
- @abstractmethod
37
- def save_session(self, session: Union[VerificationSession, EnhancedVerificationSession]) -> str:
38
- """Save a verification session. Returns session_id."""
39
- pass
40
 
41
- @abstractmethod
42
- def load_session(self, session_id: str) -> Optional[Union[VerificationSession, EnhancedVerificationSession]]:
43
- """Load a verification session by ID."""
44
- pass
45
-
46
- @abstractmethod
47
- def save_verification(
48
- self, session_id: str, record: VerificationRecord
49
- ) -> None:
50
- """Save a verification record to a session."""
51
- pass
52
-
53
- @abstractmethod
54
- def get_session_statistics(self, session_id: str) -> Dict[str, Any]:
55
- """Get statistics for a session."""
56
- pass
57
-
58
- @abstractmethod
59
- def export_to_csv(self, session_id: str) -> str:
60
- """Export session to CSV format. Returns CSV content."""
61
- pass
62
-
63
- @abstractmethod
64
- def list_sessions(self) -> List[str]:
65
- """List all session IDs."""
66
- pass
67
-
68
- @abstractmethod
69
- def delete_session(self, session_id: str) -> bool:
70
- """Delete a session. Returns True if successful."""
71
- pass
72
-
73
- @abstractmethod
74
- def get_last_session(self) -> Optional[Union[VerificationSession, EnhancedVerificationSession]]:
75
- """Get the most recently created session. Returns None if no sessions exist."""
76
- pass
77
-
78
- @abstractmethod
79
- def mark_session_complete(self, session_id: str) -> None:
80
- """Mark a session as complete and prevent further modifications."""
81
- pass
82
-
83
- @abstractmethod
84
- def can_modify_session(self, session_id: str) -> bool:
85
- """Check if a session can be modified. Returns False if session is complete."""
86
- pass
87
-
88
- # Enhanced methods for multi-mode support
89
- @abstractmethod
90
- def list_sessions_by_mode(self, mode_type: str) -> List[str]:
91
- """List session IDs filtered by mode type."""
92
- pass
93
-
94
- @abstractmethod
95
- def get_incomplete_sessions(self) -> List[Union[VerificationSession, EnhancedVerificationSession]]:
96
- """Get all incomplete sessions across all modes."""
97
- pass
98
-
99
- @abstractmethod
100
- def update_mode_metadata(self, session_id: str, metadata: Dict[str, Any]) -> None:
101
- """Update mode-specific metadata for a session."""
102
- pass
103
-
104
- @abstractmethod
105
- def export_to_xlsx(self, session_id: str) -> bytes:
106
- """Export session to XLSX format. Returns XLSX content as bytes."""
107
- pass
108
-
109
- @abstractmethod
110
- def export_to_json(self, session_id: str) -> str:
111
- """Export session to JSON format. Returns JSON content."""
112
- pass
113
-
114
- @abstractmethod
115
- def export_multiple_sessions(self, session_ids: List[str], format_type: str) -> Union[str, bytes]:
116
- """Export multiple sessions in specified format (csv, xlsx, json)."""
117
- pass
118
-
119
-
120
- class JSONVerificationStore(VerificationDataStore):
121
- """JSON-based implementation of verification data storage with enhanced multi-mode support and comprehensive error handling."""
122
-
123
- def __init__(self, storage_dir: str = ".verification_data"):
124
- """Initialize JSON store with storage directory and error handling."""
125
- self.storage_dir = Path(storage_dir)
126
- self.storage_dir.mkdir(exist_ok=True)
127
- self.sessions_dir = self.storage_dir / "sessions"
128
- self.sessions_dir.mkdir(exist_ok=True)
129
- self.edits_dir = self.storage_dir / "edits"
130
- self.edits_dir.mkdir(exist_ok=True)
131
- self.datasets_dir = self.storage_dir / "datasets"
132
- self.datasets_dir.mkdir(exist_ok=True)
133
- self.backups_dir = self.storage_dir / "backups"
134
- self.backups_dir.mkdir(exist_ok=True)
135
-
136
- # Initialize error handling (lazy initialization to avoid deepcopy issues)
137
- self._error_handler = None
138
- self._error_decorator = None
139
- self._storage_dir_str = storage_dir
140
-
141
- # Initialize data validation service
142
- self.validation_service = DataValidationService()
143
-
144
- def _get_session_path(self, session_id: str) -> Path:
145
- """Get file path for a session."""
146
- return self.sessions_dir / f"{session_id}.json"
147
-
148
- @property
149
- def error_handler(self) -> EnhancedErrorHandler:
150
- """Lazy initialization of error handler to avoid deepcopy issues."""
151
- if self._error_handler is None:
152
- self._error_handler = EnhancedErrorHandler(self._storage_dir_str)
153
- return self._error_handler
154
 
155
- @property
156
- def error_decorator(self) -> ErrorHandlingDecorator:
157
- """Lazy initialization of error decorator to avoid deepcopy issues."""
158
- if self._error_decorator is None:
159
- self._error_decorator = ErrorHandlingDecorator(self.error_handler)
160
- return self._error_decorator
161
-
162
- def save_session(self, session: Union[VerificationSession, EnhancedVerificationSession]) -> str:
163
- """Save a verification session to JSON file with automatic backup creation."""
164
- try:
165
- session_path = self._get_session_path(session.session_id)
166
- session_data = session.to_dict()
167
 
168
- # Create backup before saving (if session already exists)
169
- if session_path.exists():
170
- try:
171
- with open(session_path, "r") as f:
172
- existing_data = json.load(f)
173
- self.error_handler.recovery_manager.create_backup(session.session_id, existing_data)
174
- except Exception as e:
175
- # Log backup failure but don't fail the save
176
- logging.warning(f"Failed to create backup for session {session.session_id}: {e}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
177
 
178
- # Save the session
179
- with open(session_path, "w") as f:
180
- json.dump(session_data, f, indent=2)
181
-
182
- return session.session_id
183
 
184
- except OSError as e:
185
- if "No space left" in str(e):
186
- error_context = self.error_handler.handle_export_generation_error(
187
- "session", session.session_id, "Insufficient disk space to save session"
188
- )
189
- else:
190
- error_context = self.error_handler.handle_session_corruption_error(
191
- session.session_id, "corrupted_session", f"File system error: {str(e)}"
192
- )
193
- raise RuntimeError(error_context.user_message) from e
194
  except Exception as e:
195
- error_context = self.error_handler.handle_session_corruption_error(
196
- session.session_id, "corrupted_session", f"Unexpected error saving session: {str(e)}"
197
- )
198
- raise RuntimeError(error_context.user_message) from e
199
 
200
- def load_session(self, session_id: str) -> Optional[Union[VerificationSession, EnhancedVerificationSession]]:
201
- """Load a verification session from JSON file with corruption recovery."""
202
- session_path = self._get_session_path(session_id)
203
- if not session_path.exists():
204
- return None
205
 
 
 
 
 
 
 
 
 
 
 
 
 
206
  try:
207
- with open(session_path, "r") as f:
208
- data = json.load(f)
209
 
210
- # Validate session data integrity
211
- is_valid, validation_errors = self.error_handler.recovery_manager.validate_session_data(data)
212
- if not is_valid:
213
- # Attempt to recover from backup
214
- backups = self.error_handler.recovery_manager.list_backups(session_id)
215
- if backups:
216
- # Try the most recent backup
217
- backup_data = self.error_handler.recovery_manager.restore_from_backup(backups[0]["backup_id"])
218
- if backup_data:
219
- data = backup_data
220
- # Log the recovery
221
- logging.warning(f"Session {session_id} recovered from backup due to corruption: {validation_errors}")
222
- else:
223
- # Handle corruption error
224
- error_context = self.error_handler.handle_session_corruption_error(
225
- session_id, "corrupted_session", f"Validation errors: {validation_errors}"
226
- )
227
- raise ValueError(error_context.user_message)
228
- else:
229
- # No backups available
230
- error_context = self.error_handler.handle_session_corruption_error(
231
- session_id, "corrupted_session", f"No backups available. Validation errors: {validation_errors}"
232
- )
233
- raise ValueError(error_context.user_message)
234
 
235
- # Determine if this is an enhanced session based on presence of mode_type
236
- if "mode_type" in data:
237
- return EnhancedVerificationSession.from_dict(data)
 
238
  else:
239
- return VerificationSession.from_dict(data)
 
 
 
 
 
 
 
 
 
240
 
241
- except json.JSONDecodeError as e:
242
- # Handle JSON corruption
243
- error_context = self.error_handler.handle_session_corruption_error(
244
- session_id, "corrupted_session", f"JSON decode error: {str(e)}"
245
- )
246
 
247
- # Try to recover from backup
248
- backups = self.error_handler.recovery_manager.list_backups(session_id)
249
- if backups:
250
- backup_data = self.error_handler.recovery_manager.restore_from_backup(backups[0]["backup_id"])
251
- if backup_data:
252
- logging.warning(f"Session {session_id} recovered from backup due to JSON corruption")
253
- if "mode_type" in backup_data:
254
- return EnhancedVerificationSession.from_dict(backup_data)
255
- else:
256
- return VerificationSession.from_dict(backup_data)
257
 
258
- raise ValueError(error_context.user_message) from e
259
  except Exception as e:
260
- error_context = self.error_handler.handle_session_corruption_error(
261
- session_id, "corrupted_session", f"Unexpected error loading session: {str(e)}"
262
- )
263
- raise ValueError(error_context.user_message) from e
264
-
265
- def save_verification(
266
- self, session_id: str, record: VerificationRecord
267
- ) -> None:
268
- """Save a verification record to a session with validation."""
269
- # Validate the verification record before saving
270
- validation_result = self.validation_service.validate_verification_record(record)
271
- if not validation_result.is_valid:
272
- raise ValueError(f"Verification record validation failed: {'; '.join(validation_result.errors)}")
273
-
274
- session = self.load_session(session_id)
275
- if session is None:
276
- raise ValueError(f"Session {session_id} not found")
277
-
278
- # Prevent modifications to completed sessions
279
- if session.is_complete:
280
- raise ValueError(f"Cannot modify completed session {session_id}")
281
-
282
- # Check if record already exists and update it
283
- existing_idx = None
284
- for idx, v in enumerate(session.verifications):
285
- if v.message_id == record.message_id:
286
- existing_idx = idx
287
- break
288
-
289
- if existing_idx is not None:
290
- session.verifications[existing_idx] = record
291
- else:
292
- session.verifications.append(record)
293
-
294
- # Update counts
295
- session.verified_count = len(session.verifications)
296
- session.correct_count = sum(1 for v in session.verifications if v.is_correct)
297
- session.incorrect_count = session.verified_count - session.correct_count
298
-
299
- # Verify accuracy calculations before saving
300
- accuracy_validation = self.validation_service.verify_accuracy_calculations(session)
301
- if not accuracy_validation.is_valid:
302
- logging.warning(f"Accuracy calculation issues in session {session_id}: {'; '.join(accuracy_validation.errors)}")
303
-
304
- self.save_session(session)
305
 
306
- def get_session_statistics(self, session_id: str) -> Dict[str, Any]:
307
- """Get statistics for a session."""
308
- session = self.load_session(session_id)
309
- if session is None:
310
- raise ValueError(f"Session {session_id} not found")
311
-
312
- stats = {
313
- "session_id": session.session_id,
314
- "verifier_name": session.verifier_name,
315
- "dataset_name": session.dataset_name,
316
- "total_messages": session.total_messages,
317
- "verified_count": session.verified_count,
318
- "correct_count": session.correct_count,
319
- "incorrect_count": session.incorrect_count,
320
- "is_complete": session.is_complete,
321
- }
322
-
323
- # Calculate accuracy
324
- if session.verified_count > 0:
325
- stats["accuracy"] = (
326
- session.correct_count / session.verified_count * 100
327
- )
328
- else:
329
- stats["accuracy"] = 0.0
330
-
331
- # Calculate accuracy by type
332
- accuracy_by_type = {}
333
- for classification_type in ["green", "yellow", "red"]:
334
- type_records = [
335
- v for v in session.verifications
336
- if v.classifier_decision == classification_type
337
- ]
338
- if type_records:
339
- correct = sum(1 for v in type_records if v.is_correct)
340
- accuracy_by_type[classification_type] = (
341
- correct / len(type_records) * 100
342
- )
343
- else:
344
- accuracy_by_type[classification_type] = 0.0
345
 
346
- stats["accuracy_by_type"] = accuracy_by_type
 
 
 
347
 
348
- return stats
349
-
350
- def export_to_csv(self, session_id: str) -> str:
351
- """Export session to CSV format with comprehensive error handling."""
352
  try:
353
- session = self.load_session(session_id)
354
- if session is None:
355
- error_context = self.error_handler.handle_export_generation_error(
356
- "csv", session_id, f"Session {session_id} not found"
357
- )
358
- raise ValueError(error_context.user_message)
359
-
360
- if session.verified_count == 0:
361
- error_context = self.error_handler.handle_export_generation_error(
362
- "csv", session_id, "No verified messages to export"
363
- )
364
- raise ValueError(error_context.user_message)
365
-
366
- output = io.StringIO()
367
-
368
- # Add summary section
369
- accuracy = (
370
- session.correct_count / session.verified_count * 100
371
- if session.verified_count > 0
372
- else 0.0
373
- )
374
- output.write("VERIFICATION SUMMARY\n")
375
- output.write(f"Total Messages,{session.verified_count}\n")
376
- output.write(f"Correct,{session.correct_count}\n")
377
- output.write(f"Incorrect,{session.incorrect_count}\n")
378
- output.write(f"Accuracy %,{accuracy:.1f}\n")
379
-
380
- # Add enhanced session info if available
381
- if isinstance(session, EnhancedVerificationSession):
382
- output.write(f"Mode Type,{session.mode_type}\n")
383
- if session.file_source:
384
- output.write(f"File Source,{session.file_source}\n")
385
- if session.dataset_version:
386
- output.write(f"Dataset Version,{session.dataset_version}\n")
387
- if session.manual_input_count > 0:
388
- output.write(f"Manual Input Count,{session.manual_input_count}\n")
389
-
390
- output.write("\n")
391
-
392
- # Use CSV writer for proper escaping
393
- writer = csv.writer(output)
394
-
395
- # Add header row
396
- headers = ["Patient Message", "Classifier Said", "You Said", "Notes", "Date"]
397
- if isinstance(session, EnhancedVerificationSession):
398
- headers.extend(["Mode Type", "Confidence", "Indicators"])
399
-
400
- writer.writerow(headers)
401
-
402
- # Add data rows
403
- for record in session.verifications:
404
- row = [
405
- record.original_message,
406
- record.classifier_decision.upper(),
407
- record.ground_truth_label.upper(),
408
- record.verifier_notes,
409
- record.timestamp.strftime("%Y-%m-%d %H:%M:%S"),
410
- ]
411
-
412
- if isinstance(session, EnhancedVerificationSession):
413
- row.extend([
414
- session.mode_type,
415
- record.classifier_confidence,
416
- "; ".join(record.classifier_indicators),
417
- ])
418
-
419
- writer.writerow(row)
420
 
421
- return output.getvalue()
422
-
423
- except MemoryError as e:
424
- error_context = self.error_handler.handle_export_generation_error(
425
- "csv", session_id, f"Insufficient memory for CSV export: {str(e)}"
426
- )
427
- raise RuntimeError(error_context.user_message) from e
428
- except OSError as e:
429
- if "No space left" in str(e):
430
- error_context = self.error_handler.handle_export_generation_error(
431
- "csv", session_id, "Insufficient disk space for export"
432
- )
433
- else:
434
- error_context = self.error_handler.handle_export_generation_error(
435
- "csv", session_id, f"File system error: {str(e)}"
436
- )
437
- raise RuntimeError(error_context.user_message) from e
438
  except Exception as e:
439
- error_context = self.error_handler.handle_export_generation_error(
440
- "csv", session_id, f"Unexpected error during CSV export: {str(e)}"
441
- )
442
- raise RuntimeError(error_context.user_message) from e
443
-
444
- def list_sessions(self) -> List[str]:
445
- """List all session IDs."""
446
- session_files = self.sessions_dir.glob("*.json")
447
- return [f.stem for f in session_files]
448
-
449
- def delete_session(self, session_id: str) -> bool:
450
- """Delete a session."""
451
- session_path = self._get_session_path(session_id)
452
- if session_path.exists():
453
- session_path.unlink()
454
- return True
455
- return False
456
-
457
- def get_last_session(self) -> Optional[Union[VerificationSession, EnhancedVerificationSession]]:
458
- """Get the most recently created session."""
459
- session_files = list(self.sessions_dir.glob("*.json"))
460
- if not session_files:
461
- return None
462
-
463
- # Sort by modification time, get the most recent
464
- latest_file = max(session_files, key=lambda f: f.stat().st_mtime)
465
 
466
- with open(latest_file, "r") as f:
467
- data = json.load(f)
468
-
469
- # Determine if this is an enhanced session based on presence of mode_type
470
- if "mode_type" in data:
471
- return EnhancedVerificationSession.from_dict(data)
472
- else:
473
- return VerificationSession.from_dict(data)
474
 
475
- def mark_session_complete(self, session_id: str) -> None:
476
- """Mark a session as complete and prevent further modifications."""
477
- session = self.load_session(session_id)
478
- if session is None:
479
- raise ValueError(f"Session {session_id} not found")
480
 
481
- # Perform final validation before marking complete
482
- final_validation = self.validation_service.perform_final_session_validation(session)
483
- if not final_validation.is_valid:
484
- logging.warning(f"Session {session_id} has validation issues: {'; '.join(final_validation.errors)}")
485
- # Still allow completion but log the issues
 
 
 
486
 
487
- session.is_complete = True
488
- session.completed_at = datetime.now()
489
- self.save_session(session)
490
-
491
- def can_modify_session(self, session_id: str) -> bool:
492
- """Check if a session can be modified. Returns False if session is complete."""
493
- session = self.load_session(session_id)
494
- if session is None:
495
  return False
496
-
497
- return not session.is_complete
498
 
499
- # Enhanced methods for multi-mode support
500
- def list_sessions_by_mode(self, mode_type: str) -> List[str]:
501
- """List session IDs filtered by mode type."""
502
- session_ids = []
503
- for session_file in self.sessions_dir.glob("*.json"):
504
- try:
505
- with open(session_file, "r") as f:
506
- data = json.load(f)
507
-
508
- # Check if session has mode_type and matches filter
509
- if data.get("mode_type") == mode_type:
510
- session_ids.append(session_file.stem)
511
- elif mode_type == "standard" and "mode_type" not in data:
512
- # Include legacy sessions as "standard" mode
513
- session_ids.append(session_file.stem)
514
- except (json.JSONDecodeError, KeyError):
515
- # Skip corrupted files
516
- continue
517
-
518
- return session_ids
519
 
520
- def get_incomplete_sessions(self) -> List[Union[VerificationSession, EnhancedVerificationSession]]:
521
- """Get all incomplete sessions across all modes."""
522
- incomplete_sessions = []
523
- for session_file in self.sessions_dir.glob("*.json"):
524
- try:
525
- with open(session_file, "r") as f:
526
- data = json.load(f)
527
-
528
- # Only include incomplete sessions
529
- if not data.get("is_complete", False):
530
- if "mode_type" in data:
531
- session = EnhancedVerificationSession.from_dict(data)
532
- else:
533
- session = VerificationSession.from_dict(data)
534
- incomplete_sessions.append(session)
535
- except (json.JSONDecodeError, KeyError):
536
- # Skip corrupted files
537
- continue
538
-
539
- # Sort by creation date, most recent first
540
- incomplete_sessions.sort(key=lambda s: s.created_at, reverse=True)
541
- return incomplete_sessions
542
 
543
- def update_mode_metadata(self, session_id: str, metadata: Dict[str, Any]) -> None:
544
- """Update mode-specific metadata for a session."""
545
- session = self.load_session(session_id)
546
- if session is None:
547
- raise ValueError(f"Session {session_id} not found")
548
-
549
- # Ensure this is an enhanced session
550
- if not isinstance(session, EnhancedVerificationSession):
551
- raise ValueError(f"Session {session_id} is not an enhanced session")
552
 
553
- # Update metadata
554
- session.mode_metadata.update(metadata)
555
- self.save_session(session)
556
-
557
- def export_to_xlsx(self, session_id: str) -> bytes:
558
- """Export session to XLSX format with comprehensive error handling. Returns XLSX content as bytes."""
559
- try:
560
- try:
561
- import openpyxl
562
- from openpyxl.styles import Font, PatternFill
563
- except ImportError as e:
564
- error_context = self.error_handler.handle_export_generation_error(
565
- "xlsx", session_id, "openpyxl library not available for XLSX export"
566
- )
567
- raise ImportError(error_context.user_message) from e
568
 
569
- session = self.load_session(session_id)
570
- if session is None:
571
- error_context = self.error_handler.handle_export_generation_error(
572
- "xlsx", session_id, f"Session {session_id} not found"
573
- )
574
- raise ValueError(error_context.user_message)
575
-
576
- if session.verified_count == 0:
577
- error_context = self.error_handler.handle_export_generation_error(
578
- "xlsx", session_id, "No verified messages to export"
579
- )
580
- raise ValueError(error_context.user_message)
581
 
582
- # Create workbook with multiple sheets
583
- wb = openpyxl.Workbook()
 
 
 
 
 
 
 
584
 
585
- # Results sheet
586
- ws_results = wb.active
587
- ws_results.title = "Results"
588
-
589
- # Header styling
590
- header_font = Font(bold=True)
591
- header_fill = PatternFill(start_color="CCCCCC", end_color="CCCCCC", fill_type="solid")
592
-
593
- # Add headers
594
- headers = ["Patient Message", "Classifier Said", "You Said", "Notes", "Date"]
595
- if isinstance(session, EnhancedVerificationSession):
596
- headers.extend(["Mode Type", "Confidence", "Indicators"])
597
-
598
- for col, header in enumerate(headers, 1):
599
- cell = ws_results.cell(row=1, column=col, value=header)
600
- cell.font = header_font
601
- cell.fill = header_fill
602
-
603
- # Add data rows
604
- for row, record in enumerate(session.verifications, 2):
605
- ws_results.cell(row=row, column=1, value=record.original_message)
606
- ws_results.cell(row=row, column=2, value=record.classifier_decision.upper())
607
- ws_results.cell(row=row, column=3, value=record.ground_truth_label.upper())
608
- ws_results.cell(row=row, column=4, value=record.verifier_notes)
609
- ws_results.cell(row=row, column=5, value=record.timestamp.strftime("%Y-%m-%d %H:%M:%S"))
610
-
611
- if isinstance(session, EnhancedVerificationSession):
612
- ws_results.cell(row=row, column=6, value=session.mode_type)
613
- ws_results.cell(row=row, column=7, value=record.classifier_confidence)
614
- ws_results.cell(row=row, column=8, value="; ".join(record.classifier_indicators))
615
-
616
- # Summary Statistics sheet
617
- ws_summary = wb.create_sheet("Summary Statistics")
618
 
619
- # Calculate statistics
620
- accuracy = (session.correct_count / session.verified_count * 100) if session.verified_count > 0 else 0.0
621
-
622
- summary_data = [
623
- ["Metric", "Value"],
624
- ["Session ID", session.session_id],
625
- ["Verifier Name", session.verifier_name],
626
- ["Dataset Name", session.dataset_name],
627
- ["Total Messages", session.verified_count],
628
- ["Correct", session.correct_count],
629
- ["Incorrect", session.incorrect_count],
630
- ["Accuracy %", f"{accuracy:.1f}%"],
631
- ["Created At", session.created_at.strftime("%Y-%m-%d %H:%M:%S")],
632
- ["Completed At", session.completed_at.strftime("%Y-%m-%d %H:%M:%S") if session.completed_at else "In Progress"],
633
- ]
634
-
635
- if isinstance(session, EnhancedVerificationSession):
636
- summary_data.extend([
637
- ["Mode Type", session.mode_type],
638
- ["File Source", session.file_source or "N/A"],
639
- ["Dataset Version", session.dataset_version or "N/A"],
640
- ["Manual Input Count", session.manual_input_count],
641
- ])
642
 
643
- for row, (metric, value) in enumerate(summary_data, 1):
644
- cell_metric = ws_summary.cell(row=row, column=1, value=metric)
645
- cell_value = ws_summary.cell(row=row, column=2, value=value)
646
- if row == 1: # Header row
647
- cell_metric.font = header_font
648
- cell_metric.fill = header_fill
649
- cell_value.font = header_font
650
- cell_value.fill = header_fill
651
-
652
- # Error Analysis sheet
653
- ws_errors = wb.create_sheet("Error Analysis")
654
-
655
- # Group errors by classification type
656
- error_analysis = {}
657
- for record in session.verifications:
658
- if not record.is_correct:
659
- key = f"{record.classifier_decision} -> {record.ground_truth_label}"
660
- if key not in error_analysis:
661
- error_analysis[key] = []
662
- error_analysis[key].append(record)
663
 
664
- error_headers = ["Error Type", "Count", "Example Message", "Notes"]
665
- for col, header in enumerate(error_headers, 1):
666
- cell = ws_errors.cell(row=1, column=col, value=header)
667
- cell.font = header_font
668
- cell.fill = header_fill
669
 
670
- row = 2
671
- for error_type, records in error_analysis.items():
672
- ws_errors.cell(row=row, column=1, value=error_type)
673
- ws_errors.cell(row=row, column=2, value=len(records))
674
- ws_errors.cell(row=row, column=3, value=records[0].original_message[:100] + "..." if len(records[0].original_message) > 100 else records[0].original_message)
675
- ws_errors.cell(row=row, column=4, value=records[0].verifier_notes)
676
- row += 1
677
 
678
- # Auto-adjust column widths
679
- for ws in [ws_results, ws_summary, ws_errors]:
680
- for column in ws.columns:
681
- max_length = 0
682
- column_letter = column[0].column_letter
683
- for cell in column:
684
- try:
685
- if len(str(cell.value)) > max_length:
686
- max_length = len(str(cell.value))
687
- except:
688
- pass
689
- adjusted_width = min(max_length + 2, 50) # Cap at 50 characters
690
- ws.column_dimensions[column_letter].width = adjusted_width
691
-
692
- # Save to bytes
693
- output = io.BytesIO()
694
- wb.save(output)
695
- output.seek(0)
696
- return output.getvalue()
697
 
698
- except MemoryError as e:
699
- error_context = self.error_handler.handle_export_generation_error(
700
- "xlsx", session_id, f"Insufficient memory for XLSX export: {str(e)}"
701
- )
702
- raise RuntimeError(error_context.user_message) from e
703
- except OSError as e:
704
- if "No space left" in str(e):
705
- error_context = self.error_handler.handle_export_generation_error(
706
- "xlsx", session_id, "Insufficient disk space for export"
707
- )
708
- else:
709
- error_context = self.error_handler.handle_export_generation_error(
710
- "xlsx", session_id, f"File system error: {str(e)}"
711
- )
712
- raise RuntimeError(error_context.user_message) from e
713
  except Exception as e:
714
- error_context = self.error_handler.handle_export_generation_error(
715
- "xlsx", session_id, f"Unexpected error during XLSX export: {str(e)}"
716
- )
717
- raise RuntimeError(error_context.user_message) from e
718
 
719
- def export_to_json(self, session_id: str) -> str:
720
- """Export session to JSON format with comprehensive error handling. Returns JSON content."""
721
- try:
722
- session = self.load_session(session_id)
723
- if session is None:
724
- error_context = self.error_handler.handle_export_generation_error(
725
- "json", session_id, f"Session {session_id} not found"
726
- )
727
- raise ValueError(error_context.user_message)
728
-
729
- # Create comprehensive export data
730
- export_data = {
731
- "export_metadata": {
732
- "export_timestamp": datetime.now().isoformat(),
733
- "session_id": session_id,
734
- "export_format": "json",
735
- "version": "1.0"
736
- },
737
- "session_data": session.to_dict(),
738
- "statistics": self.get_session_statistics(session_id),
739
- }
740
-
741
- # Add enhanced data if available
742
- if isinstance(session, EnhancedVerificationSession):
743
- export_data["enhanced_metadata"] = {
744
- "mode_type": session.mode_type,
745
- "mode_metadata": session.mode_metadata,
746
- "file_source": session.file_source,
747
- "dataset_version": session.dataset_version,
748
- "manual_input_count": session.manual_input_count,
749
- }
750
 
751
- return json.dumps(export_data, indent=2)
 
752
 
753
- except MemoryError as e:
754
- error_context = self.error_handler.handle_export_generation_error(
755
- "json", session_id, f"Insufficient memory for JSON export: {str(e)}"
756
- )
757
- raise RuntimeError(error_context.user_message) from e
758
- except TypeError as e:
759
- error_context = self.error_handler.handle_export_generation_error(
760
- "json", session_id, f"Data serialization error: {str(e)}"
761
- )
762
- raise RuntimeError(error_context.user_message) from e
763
- except Exception as e:
764
- error_context = self.error_handler.handle_export_generation_error(
765
- "json", session_id, f"Unexpected error during JSON export: {str(e)}"
766
- )
767
- raise RuntimeError(error_context.user_message) from e
768
-
769
- def export_multiple_sessions(self, session_ids: List[str], format_type: str) -> Union[str, bytes]:
770
- """Export multiple sessions in specified format (csv, xlsx, json)."""
771
- if not session_ids:
772
- raise ValueError("No session IDs provided")
773
-
774
- if format_type.lower() == "csv":
775
- return self._export_multiple_sessions_csv(session_ids)
776
- elif format_type.lower() == "xlsx":
777
- return self._export_multiple_sessions_xlsx(session_ids)
778
- elif format_type.lower() == "json":
779
- return self._export_multiple_sessions_json(session_ids)
780
- else:
781
- raise ValueError(f"Unsupported format type: {format_type}")
782
-
783
- def _export_multiple_sessions_csv(self, session_ids: List[str]) -> str:
784
- """Export multiple sessions to CSV format."""
785
- output = io.StringIO()
786
- writer = csv.writer(output)
787
-
788
- # Write combined header
789
- writer.writerow([
790
- "Session ID", "Mode Type", "Patient Message", "Classifier Said",
791
- "You Said", "Notes", "Date", "Verifier Name", "Dataset Name"
792
- ])
793
 
794
- for session_id in session_ids:
795
  session = self.load_session(session_id)
796
- if session is None:
797
- continue
 
798
 
799
- mode_type = session.mode_type if isinstance(session, EnhancedVerificationSession) else "standard"
 
 
800
 
801
- for record in session.verifications:
802
- writer.writerow([
803
- session.session_id,
804
- mode_type,
805
- record.original_message,
806
- record.classifier_decision.upper(),
807
- record.ground_truth_label.upper(),
808
- record.verifier_notes,
809
- record.timestamp.strftime("%Y-%m-%d %H:%M:%S"),
810
- session.verifier_name,
811
- session.dataset_name,
812
- ])
813
-
814
- return output.getvalue()
815
-
816
- def _export_multiple_sessions_xlsx(self, session_ids: List[str]) -> bytes:
817
- """Export multiple sessions to XLSX format."""
818
- try:
819
- import openpyxl
820
- from openpyxl.styles import Font, PatternFill
821
- except ImportError:
822
- raise ImportError("openpyxl is required for XLSX export. Install with: pip install openpyxl")
823
-
824
- wb = openpyxl.Workbook()
825
- ws = wb.active
826
- ws.title = "Combined Results"
827
-
828
- # Header styling
829
- header_font = Font(bold=True)
830
- header_fill = PatternFill(start_color="CCCCCC", end_color="CCCCCC", fill_type="solid")
831
-
832
- # Add headers
833
- headers = [
834
- "Session ID", "Mode Type", "Patient Message", "Classifier Said",
835
- "You Said", "Notes", "Date", "Verifier Name", "Dataset Name"
836
- ]
837
-
838
- for col, header in enumerate(headers, 1):
839
- cell = ws.cell(row=1, column=col, value=header)
840
- cell.font = header_font
841
- cell.fill = header_fill
842
-
843
- # Add data from all sessions
844
- row = 2
845
- for session_id in session_ids:
846
- session = self.load_session(session_id)
847
- if session is None:
848
- continue
849
 
850
- mode_type = session.mode_type if isinstance(session, EnhancedVerificationSession) else "standard"
 
 
851
 
852
- for record in session.verifications:
853
- ws.cell(row=row, column=1, value=session.session_id)
854
- ws.cell(row=row, column=2, value=mode_type)
855
- ws.cell(row=row, column=3, value=record.original_message)
856
- ws.cell(row=row, column=4, value=record.classifier_decision.upper())
857
- ws.cell(row=row, column=5, value=record.ground_truth_label.upper())
858
- ws.cell(row=row, column=6, value=record.verifier_notes)
859
- ws.cell(row=row, column=7, value=record.timestamp.strftime("%Y-%m-%d %H:%M:%S"))
860
- ws.cell(row=row, column=8, value=session.verifier_name)
861
- ws.cell(row=row, column=9, value=session.dataset_name)
862
- row += 1
863
-
864
- # Auto-adjust column widths
865
- for column in ws.columns:
866
- max_length = 0
867
- column_letter = column[0].column_letter
868
- for cell in column:
869
- try:
870
- if len(str(cell.value)) > max_length:
871
- max_length = len(str(cell.value))
872
- except:
873
- pass
874
- adjusted_width = min(max_length + 2, 50)
875
- ws.column_dimensions[column_letter].width = adjusted_width
876
-
877
- # Save to bytes
878
- output = io.BytesIO()
879
- wb.save(output)
880
- output.seek(0)
881
- return output.getvalue()
882
-
883
- def _export_multiple_sessions_json(self, session_ids: List[str]) -> str:
884
- """Export multiple sessions to JSON format."""
885
- export_data = {
886
- "export_metadata": {
887
- "export_timestamp": datetime.now().isoformat(),
888
- "session_count": len(session_ids),
889
- "export_format": "json",
890
- "version": "1.0"
891
- },
892
- "sessions": []
893
- }
894
-
895
- for session_id in session_ids:
896
- session = self.load_session(session_id)
897
- if session is None:
898
- continue
899
 
900
- session_export = {
901
- "session_data": session.to_dict(),
902
- "statistics": self.get_session_statistics(session_id),
903
- }
 
904
 
905
- if isinstance(session, EnhancedVerificationSession):
906
- session_export["enhanced_metadata"] = {
907
- "mode_type": session.mode_type,
908
- "mode_metadata": session.mode_metadata,
909
- "file_source": session.file_source,
910
- "dataset_version": session.dataset_version,
911
- "manual_input_count": session.manual_input_count,
912
- }
913
 
914
- export_data["sessions"].append(session_export)
915
-
916
- return json.dumps(export_data, indent=2)
917
-
918
- # Helper methods for enhanced functionality
919
- def save_test_case_edit(self, edit: TestCaseEdit) -> str:
920
- """Save a test case edit record."""
921
- edit_path = self.edits_dir / f"{edit.edit_id}.json"
922
- with open(edit_path, "w") as f:
923
- json.dump(edit.to_dict(), f, indent=2)
924
- return edit.edit_id
925
-
926
- def load_test_case_edit(self, edit_id: str) -> Optional[TestCaseEdit]:
927
- """Load a test case edit record."""
928
- edit_path = self.edits_dir / f"{edit_id}.json"
929
- if not edit_path.exists():
930
- return None
931
-
932
- with open(edit_path, "r") as f:
933
- data = json.load(f)
934
-
935
- return TestCaseEdit.from_dict(data)
936
-
937
- def list_test_case_edits(self, test_case_id: str = None) -> List[TestCaseEdit]:
938
- """List test case edits, optionally filtered by test case ID."""
939
- edits = []
940
- for edit_file in self.edits_dir.glob("*.json"):
941
- try:
942
- with open(edit_file, "r") as f:
943
- data = json.load(f)
944
-
945
- edit = TestCaseEdit.from_dict(data)
946
- if test_case_id is None or edit.test_case_id == test_case_id:
947
- edits.append(edit)
948
- except (json.JSONDecodeError, KeyError):
949
- continue
950
 
951
- # Sort by timestamp, most recent first
952
- edits.sort(key=lambda e: e.timestamp, reverse=True)
953
- return edits
954
-
955
- def save_file_upload_result(self, result: FileUploadResult) -> str:
956
- """Save a file upload result."""
957
- result_path = self.storage_dir / f"upload_{result.file_id}.json"
958
- with open(result_path, "w") as f:
959
- json.dump(result.to_dict(), f, indent=2)
960
- return result.file_id
961
 
962
- def load_file_upload_result(self, file_id: str) -> Optional[FileUploadResult]:
963
- """Load a file upload result."""
964
- result_path = self.storage_dir / f"upload_{file_id}.json"
965
- if not result_path.exists():
966
- return None
967
-
968
- with open(result_path, "r") as f:
969
- data = json.load(f)
970
-
971
- return FileUploadResult.from_dict(data)
972
- def get_error_recovery_options(self, error_id: str) -> List[Dict[str, Any]]:
973
- """Get recovery options for a storage error."""
974
- return self.error_handler.get_recovery_options(error_id)
975
-
976
- def attempt_error_recovery(self, error_id: str, strategy: str,
977
- recovery_data: Optional[Dict[str, Any]] = None) -> Tuple[bool, str]:
978
- """Attempt to recover from a storage error."""
979
- from src.core.enhanced_error_handler import RecoveryStrategy
980
 
981
- try:
982
- strategy_enum = RecoveryStrategy(strategy)
983
- return self.error_handler.attempt_recovery(error_id, strategy_enum, recovery_data)
984
- except ValueError:
985
- return False, f"Invalid recovery strategy: {strategy}"
986
 
987
- def restore_session_from_backup(self, session_id: str, backup_id: Optional[str] = None) -> bool:
988
- """Restore a session from backup."""
 
989
  try:
990
- backups = self.error_handler.recovery_manager.list_backups(session_id)
991
- if not backups:
992
- return False
993
-
994
- # Use specified backup or most recent
995
- target_backup_id = backup_id or backups[0]["backup_id"]
996
 
997
- restored_data = self.error_handler.recovery_manager.restore_from_backup(target_backup_id)
998
- if not restored_data:
999
- return False
1000
-
1001
- # Validate restored data
1002
- is_valid, validation_errors = self.error_handler.recovery_manager.validate_session_data(restored_data)
1003
- if not is_valid:
1004
- logging.error(f"Restored backup data is invalid: {validation_errors}")
1005
- return False
1006
-
1007
- # Save restored session
1008
- session_path = self._get_session_path(session_id)
1009
- with open(session_path, "w") as f:
1010
- json.dump(restored_data, f, indent=2)
1011
-
1012
- logging.info(f"Successfully restored session {session_id} from backup {target_backup_id}")
1013
- return True
1014
 
1015
- except Exception as e:
1016
- logging.error(f"Failed to restore session {session_id} from backup: {e}")
1017
- return False
1018
 
1019
- def list_session_backups(self, session_id: str) -> List[Dict[str, Any]]:
1020
- """List available backups for a session."""
1021
- return self.error_handler.recovery_manager.list_backups(session_id)
1022
-
1023
- def validate_session_integrity(self, session_id: str) -> Tuple[bool, List[str]]:
1024
- """Validate the integrity of a session."""
1025
- try:
1026
- session_path = self._get_session_path(session_id)
1027
- if not session_path.exists():
1028
- return False, ["Session file does not exist"]
1029
-
1030
- with open(session_path, "r") as f:
1031
- data = json.load(f)
1032
-
1033
- return self.error_handler.recovery_manager.validate_session_data(data)
1034
-
1035
- except json.JSONDecodeError as e:
1036
- return False, [f"JSON decode error: {str(e)}"]
1037
- except Exception as e:
1038
- return False, [f"Error validating session: {str(e)}"]
1039
-
1040
- def get_error_summary(self, time_window_hours: int = 24) -> Dict[str, Any]:
1041
- """Get error summary for the storage system."""
1042
- return self.error_handler.get_error_summary(time_window_hours)
1043
-
1044
- def cleanup_old_errors(self, days_to_keep: int = 7) -> int:
1045
- """Clean up old resolved errors."""
1046
- return self.error_handler.cleanup_old_errors(days_to_keep)
1047
-
1048
- # Data validation and integrity methods
1049
-
1050
- def validate_session_data_integrity(self, session_id: str) -> Dict[str, Any]:
1051
- """
1052
- Validate the data integrity of a session.
1053
-
1054
- Requirements: 11.1, 11.2, 11.5 - Verification result validation, accuracy verification, final validation
1055
- """
1056
- session = self.load_session(session_id)
1057
- if session is None:
1058
- return {"valid": False, "error": f"Session {session_id} not found"}
1059
-
1060
- # Perform comprehensive validation
1061
- session_validation = self.validation_service.validate_verification_session(session)
1062
- accuracy_validation = self.validation_service.verify_accuracy_calculations(session)
1063
-
1064
- # Generate integrity checksum
1065
- integrity_checksum = self.validation_service.generate_data_integrity_checksum(session)
1066
-
1067
- return {
1068
- "valid": session_validation.is_valid and accuracy_validation.is_valid,
1069
- "session_validation": {
1070
- "valid": session_validation.is_valid,
1071
- "errors": session_validation.errors,
1072
- "warnings": session_validation.warnings
1073
- },
1074
- "accuracy_validation": {
1075
- "valid": accuracy_validation.is_valid,
1076
- "errors": accuracy_validation.errors,
1077
- "warnings": accuracy_validation.warnings,
1078
- "metadata": accuracy_validation.metadata
1079
- },
1080
- "integrity_checksum": {
1081
- "checksum": integrity_checksum.checksum_value,
1082
- "timestamp": integrity_checksum.timestamp.isoformat(),
1083
- "data_size": integrity_checksum.data_size
1084
- }
1085
- }
1086
-
1087
- def detect_duplicate_test_cases_in_import(self, test_cases: List[TestMessage],
1088
- similarity_threshold: float = 0.95) -> Dict[str, Any]:
1089
- """
1090
- Detect duplicate test cases in import data.
1091
-
1092
- Requirements: 11.4 - Duplicate detection for test case imports
1093
- """
1094
- # Validate individual test messages first
1095
- validation_results = []
1096
- valid_test_cases = []
1097
-
1098
- for i, test_case in enumerate(test_cases):
1099
- validation = self.validation_service.validate_test_message(test_case)
1100
- validation_results.append({
1101
- "index": i,
1102
- "message_id": test_case.message_id,
1103
- "valid": validation.is_valid,
1104
- "errors": validation.errors,
1105
- "warnings": validation.warnings
1106
- })
1107
 
1108
- if validation.is_valid:
1109
- valid_test_cases.append(test_case)
1110
-
1111
- # Detect duplicates among valid test cases
1112
- duplicate_result = self.validation_service.detect_duplicate_test_cases(
1113
- valid_test_cases, similarity_threshold
1114
- )
1115
-
1116
- return {
1117
- "total_test_cases": len(test_cases),
1118
- "valid_test_cases": len(valid_test_cases),
1119
- "validation_results": validation_results,
1120
- "duplicate_detection": {
1121
- "duplicates_found": duplicate_result.duplicates_found,
1122
- "duplicate_groups": duplicate_result.duplicate_groups,
1123
- "similarity_threshold": duplicate_result.similarity_threshold,
1124
- "detection_method": duplicate_result.detection_method
1125
- }
1126
- }
1127
-
1128
- def export_with_integrity_checksum(self, session_id: str, format_type: str) -> Dict[str, Any]:
1129
- """
1130
- Export session data with integrity checksum for validation.
1131
-
1132
- Requirements: 11.3 - Data integrity checksums for exports
1133
- """
1134
- session = self.load_session(session_id)
1135
- if session is None:
1136
- raise ValueError(f"Session {session_id} not found")
1137
-
1138
- # Generate export data
1139
- if format_type.lower() == "csv":
1140
- export_data = self.export_to_csv(session_id)
1141
- elif format_type.lower() == "xlsx":
1142
- export_data = self.export_to_xlsx(session_id)
1143
- elif format_type.lower() == "json":
1144
- export_data = self.export_to_json(session_id)
1145
- else:
1146
- raise ValueError(f"Unsupported export format: {format_type}")
1147
-
1148
- # Generate integrity checksum for the export
1149
- export_checksum = self.validation_service.generate_data_integrity_checksum(
1150
- export_data,
1151
- validation_fields=["session_id", "verifications", "statistics"]
1152
- )
1153
-
1154
- # Generate session integrity checksum
1155
- session_checksum = self.validation_service.generate_data_integrity_checksum(session)
1156
-
1157
- return {
1158
- "export_data": export_data,
1159
- "export_metadata": {
1160
- "session_id": session_id,
1161
- "format_type": format_type,
1162
- "export_timestamp": datetime.now().isoformat(),
1163
- "export_checksum": {
1164
- "checksum": export_checksum.checksum_value,
1165
- "checksum_type": export_checksum.checksum_type,
1166
- "data_size": export_checksum.data_size,
1167
- "validation_fields": export_checksum.validation_fields
1168
- },
1169
- "session_checksum": {
1170
- "checksum": session_checksum.checksum_value,
1171
- "checksum_type": session_checksum.checksum_type,
1172
- "data_size": session_checksum.data_size
1173
- }
1174
- }
1175
- }
1176
-
1177
- def validate_import_data_integrity(self, import_data: Any, expected_checksum: str,
1178
- checksum_type: str = "sha256") -> Dict[str, Any]:
1179
- """
1180
- Validate imported data against expected integrity checksum.
1181
-
1182
- Requirements: 11.3 - Data integrity checksums for exports
1183
- """
1184
- from src.core.data_validation_service import IntegrityChecksum
1185
-
1186
- expected_checksum_obj = IntegrityChecksum(
1187
- checksum_type=checksum_type,
1188
- checksum_value=expected_checksum,
1189
- data_size=0, # Will be recalculated
1190
- timestamp=datetime.now(),
1191
- validation_fields=[]
1192
- )
1193
-
1194
- validation_result = self.validation_service.validate_data_integrity(
1195
- import_data, expected_checksum_obj
1196
- )
1197
-
1198
- return {
1199
- "valid": validation_result.is_valid,
1200
- "errors": validation_result.errors,
1201
- "warnings": validation_result.warnings,
1202
- "metadata": validation_result.metadata
1203
- }
1204
-
1205
- def get_session_data_quality_report(self, session_id: str) -> Dict[str, Any]:
1206
- """
1207
- Generate comprehensive data quality report for a session.
1208
-
1209
- Requirements: 11.5 - Final session validation checks
1210
- """
1211
- session = self.load_session(session_id)
1212
- if session is None:
1213
- return {"error": f"Session {session_id} not found"}
1214
-
1215
- # Perform final validation
1216
- final_validation = self.validation_service.perform_final_session_validation(session)
1217
-
1218
- # Get session statistics
1219
- stats = self.get_session_statistics(session_id)
1220
-
1221
- # Calculate additional quality metrics
1222
- quality_metrics = {}
1223
- if hasattr(session, 'verifications') and session.verifications:
1224
- # Calculate completeness metrics
1225
- records_with_notes = sum(1 for v in session.verifications
1226
- if hasattr(v, 'verifier_notes') and v.verifier_notes.strip())
1227
- quality_metrics["notes_completeness"] = records_with_notes / len(session.verifications)
1228
 
1229
- # Calculate confidence distribution
1230
- confidences = [v.classifier_confidence for v in session.verifications
1231
- if hasattr(v, 'classifier_confidence')]
1232
- if confidences:
1233
- quality_metrics["avg_confidence"] = sum(confidences) / len(confidences)
1234
- quality_metrics["min_confidence"] = min(confidences)
1235
- quality_metrics["max_confidence"] = max(confidences)
1236
-
1237
- return {
1238
- "session_id": session_id,
1239
- "report_timestamp": datetime.now().isoformat(),
1240
- "validation_result": {
1241
- "valid": final_validation.is_valid,
1242
- "errors": final_validation.errors,
1243
- "warnings": final_validation.warnings,
1244
- "data_quality_score": final_validation.metadata.get("data_quality_score", 0)
1245
- },
1246
- "session_statistics": stats,
1247
- "quality_metrics": quality_metrics,
1248
- "integrity_checksum": final_validation.metadata.get("integrity_checksum", "")
1249
- }
 
1
+ #!/usr/bin/env python3
2
  """
3
+ Verification Session Storage and Persistence.
4
 
5
+ Handles saving, loading, and managing verification sessions with recovery capabilities.
 
6
  """
7
 
8
  import json
9
  import os
10
+ import glob
 
 
 
 
11
  from datetime import datetime
12
+ from typing import List, Optional, Dict, Any
13
+ from dataclasses import asdict
14
 
15
+ from src.core.conversation_verification import VerificationSession, VerificationRecord
 
 
 
 
 
 
 
 
 
 
 
16
 
17
 
18
+ class JSONVerificationStore:
19
+ """JSON-based storage for verification sessions."""
20
 
21
+ def __init__(self, storage_dir: str = "verification_sessions"):
22
+ """Initialize verification store."""
23
+ self.storage_dir = storage_dir
24
+ os.makedirs(storage_dir, exist_ok=True)
25
 
26
+ def save_session(self, session: VerificationSession) -> bool:
27
+ """
28
+ Save verification session to storage.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
29
 
30
+ Args:
31
+ session: VerificationSession to save
 
 
 
 
 
 
 
 
 
 
32
 
33
+ Returns:
34
+ True if saved successfully, False otherwise
35
+ """
36
+ try:
37
+ filename = f"{session.session_id}.json"
38
+ filepath = os.path.join(self.storage_dir, filename)
39
+
40
+ # Convert to dict for JSON serialization
41
+ session_dict = asdict(session)
42
+
43
+ # Convert datetime objects to ISO strings
44
+ session_dict['start_time'] = session.start_time.isoformat()
45
+ if session.end_time:
46
+ session_dict['end_time'] = session.end_time.isoformat()
47
+
48
+ for record in session_dict['verification_records']:
49
+ if isinstance(record['timestamp'], datetime):
50
+ record['timestamp'] = record['timestamp'].isoformat()
51
+ if record['verification_timestamp'] and isinstance(record['verification_timestamp'], datetime):
52
+ record['verification_timestamp'] = record['verification_timestamp'].isoformat()
53
+
54
+ # Add metadata for recovery
55
+ session_dict['_metadata'] = {
56
+ 'saved_at': datetime.now().isoformat(),
57
+ 'version': '1.0',
58
+ 'storage_format': 'json'
59
+ }
60
+
61
+ with open(filepath, 'w', encoding='utf-8') as f:
62
+ json.dump(session_dict, f, ensure_ascii=False, indent=2)
63
 
64
+ return True
 
 
 
 
65
 
 
 
 
 
 
 
 
 
 
 
66
  except Exception as e:
67
+ print(f"Error saving verification session {session.session_id}: {e}")
68
+ return False
 
 
69
 
70
+ def load_session(self, session_id: str) -> Optional[VerificationSession]:
71
+ """
72
+ Load verification session from storage.
 
 
73
 
74
+ Args:
75
+ session_id: ID of session to load
76
+
77
+ Returns:
78
+ VerificationSession if found and valid, None otherwise
79
+ """
80
+ filename = f"{session_id}.json"
81
+ filepath = os.path.join(self.storage_dir, filename)
82
+
83
+ if not os.path.exists(filepath):
84
+ return None
85
+
86
  try:
87
+ with open(filepath, 'r', encoding='utf-8') as f:
88
+ session_dict = json.load(f)
89
 
90
+ # Remove metadata if present
91
+ session_dict.pop('_metadata', None)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
92
 
93
+ # Convert ISO strings back to datetime objects
94
+ session_dict['start_time'] = datetime.fromisoformat(session_dict['start_time'])
95
+ if session_dict.get('end_time'):
96
+ session_dict['end_time'] = datetime.fromisoformat(session_dict['end_time'])
97
  else:
98
+ session_dict['end_time'] = None
99
+
100
+ # Convert verification records
101
+ verification_records = []
102
+ for record_dict in session_dict['verification_records']:
103
+ record_dict['timestamp'] = datetime.fromisoformat(record_dict['timestamp'])
104
+ if record_dict.get('verification_timestamp'):
105
+ record_dict['verification_timestamp'] = datetime.fromisoformat(record_dict['verification_timestamp'])
106
+ else:
107
+ record_dict['verification_timestamp'] = None
108
 
109
+ verification_records.append(VerificationRecord(**record_dict))
110
+
111
+ session_dict['verification_records'] = verification_records
 
 
112
 
113
+ return VerificationSession(**session_dict)
 
 
 
 
 
 
 
 
 
114
 
 
115
  except Exception as e:
116
+ print(f"Error loading verification session {session_id}: {e}")
117
+ return None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
118
 
119
+ def list_sessions(self) -> List[Dict[str, Any]]:
120
+ """
121
+ List all verification sessions with basic info.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
122
 
123
+ Returns:
124
+ List of session info dictionaries
125
+ """
126
+ sessions = []
127
 
 
 
 
 
128
  try:
129
+ pattern = os.path.join(self.storage_dir, "verification_*.json")
130
+ for filepath in glob.glob(pattern):
131
+ try:
132
+ with open(filepath, 'r', encoding='utf-8') as f:
133
+ session_dict = json.load(f)
134
+
135
+ # Extract basic info without loading full session
136
+ session_info = {
137
+ 'session_id': session_dict['session_id'],
138
+ 'patient_name': session_dict['patient_name'],
139
+ 'verifier_name': session_dict['verifier_name'],
140
+ 'start_time': session_dict['start_time'],
141
+ 'end_time': session_dict.get('end_time'),
142
+ 'is_complete': session_dict['is_complete'],
143
+ 'total_exchanges': session_dict['total_exchanges'],
144
+ 'verified_exchanges': session_dict['verified_exchanges'],
145
+ 'file_path': filepath
146
+ }
147
+ sessions.append(session_info)
148
+
149
+ except Exception as e:
150
+ print(f"Error reading session file {filepath}: {e}")
151
+ continue
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
152
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
153
  except Exception as e:
154
+ print(f"Error listing sessions: {e}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
155
 
156
+ # Sort by start time (newest first)
157
+ sessions.sort(key=lambda x: x['start_time'], reverse=True)
158
+ return sessions
 
 
 
 
 
159
 
160
+ def delete_session(self, session_id: str) -> bool:
161
+ """
162
+ Delete verification session from storage.
 
 
163
 
164
+ Args:
165
+ session_id: ID of session to delete
166
+
167
+ Returns:
168
+ True if deleted successfully, False otherwise
169
+ """
170
+ filename = f"{session_id}.json"
171
+ filepath = os.path.join(self.storage_dir, filename)
172
 
173
+ try:
174
+ if os.path.exists(filepath):
175
+ os.remove(filepath)
176
+ return True
177
+ return False
178
+ except Exception as e:
179
+ print(f"Error deleting session {session_id}: {e}")
 
180
  return False
 
 
181
 
182
+ def session_exists(self, session_id: str) -> bool:
183
+ """Check if session exists in storage."""
184
+ filename = f"{session_id}.json"
185
+ filepath = os.path.join(self.storage_dir, filename)
186
+ return os.path.exists(filepath)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
187
 
188
+ def get_incomplete_sessions(self) -> List[Dict[str, Any]]:
189
+ """Get list of incomplete verification sessions."""
190
+ all_sessions = self.list_sessions()
191
+ return [s for s in all_sessions if not s['is_complete']]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
192
 
193
+ def cleanup_old_sessions(self, days_old: int = 30) -> int:
194
+ """
195
+ Clean up old completed sessions.
 
 
 
 
 
 
196
 
197
+ Args:
198
+ days_old: Delete sessions older than this many days
 
 
 
 
 
 
 
 
 
 
 
 
 
199
 
200
+ Returns:
201
+ Number of sessions deleted
202
+ """
203
+ deleted_count = 0
204
+ cutoff_date = datetime.now().timestamp() - (days_old * 24 * 60 * 60)
 
 
 
 
 
 
 
205
 
206
+ try:
207
+ for session_info in self.list_sessions():
208
+ if session_info['is_complete']:
209
+ session_date = datetime.fromisoformat(session_info['start_time']).timestamp()
210
+ if session_date < cutoff_date:
211
+ if self.delete_session(session_info['session_id']):
212
+ deleted_count += 1
213
+ except Exception as e:
214
+ print(f"Error during cleanup: {e}")
215
 
216
+ return deleted_count
217
+
218
+ def backup_session(self, session_id: str, backup_dir: str = "verification_backups") -> bool:
219
+ """
220
+ Create backup copy of session.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
221
 
222
+ Args:
223
+ session_id: ID of session to backup
224
+ backup_dir: Directory for backups
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
225
 
226
+ Returns:
227
+ True if backup created successfully
228
+ """
229
+ try:
230
+ os.makedirs(backup_dir, exist_ok=True)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
231
 
232
+ source_file = os.path.join(self.storage_dir, f"{session_id}.json")
233
+ backup_file = os.path.join(backup_dir, f"{session_id}_backup_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json")
 
 
 
234
 
235
+ if os.path.exists(source_file):
236
+ import shutil
237
+ shutil.copy2(source_file, backup_file)
238
+ return True
 
 
 
239
 
240
+ return False
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
241
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
242
  except Exception as e:
243
+ print(f"Error creating backup for session {session_id}: {e}")
244
+ return False
 
 
245
 
246
+ def validate_session_integrity(self, session_id: str) -> Dict[str, Any]:
247
+ """
248
+ Validate session data integrity.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
249
 
250
+ Args:
251
+ session_id: ID of session to validate
252
 
253
+ Returns:
254
+ Dictionary with validation results
255
+ """
256
+ validation_result = {
257
+ 'is_valid': False,
258
+ 'errors': [],
259
+ 'warnings': [],
260
+ 'session_id': session_id
261
+ }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
262
 
263
+ try:
264
  session = self.load_session(session_id)
265
+ if not session:
266
+ validation_result['errors'].append('Session not found or could not be loaded')
267
+ return validation_result
268
 
269
+ # Check basic integrity
270
+ if len(session.verification_records) != session.total_exchanges:
271
+ validation_result['errors'].append(f'Record count mismatch: {len(session.verification_records)} != {session.total_exchanges}')
272
 
273
+ # Check verified count consistency
274
+ actual_verified = len([r for r in session.verification_records if r.is_correct is not None])
275
+ if actual_verified != session.verified_exchanges:
276
+ validation_result['errors'].append(f'Verified count mismatch: {actual_verified} != {session.verified_exchanges}')
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
277
 
278
+ # Check completion status consistency
279
+ if session.is_complete and session.verified_exchanges != session.total_exchanges:
280
+ validation_result['errors'].append('Session marked complete but not all exchanges verified')
281
 
282
+ # Check for duplicate exchange IDs
283
+ exchange_ids = [r.exchange_id for r in session.verification_records]
284
+ if len(exchange_ids) != len(set(exchange_ids)):
285
+ validation_result['errors'].append('Duplicate exchange IDs found')
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
286
 
287
+ # Warnings for potential issues
288
+ if session.verified_exchanges > 0:
289
+ progress = session.get_progress()
290
+ if progress.accuracy_overall < 0.5:
291
+ validation_result['warnings'].append('Low accuracy detected - may indicate systematic issues')
292
 
293
+ validation_result['is_valid'] = len(validation_result['errors']) == 0
 
 
 
 
 
 
 
294
 
295
+ except Exception as e:
296
+ validation_result['errors'].append(f'Validation error: {str(e)}')
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
297
 
298
+ return validation_result
 
 
 
 
 
 
 
 
 
299
 
300
+ def recover_corrupted_session(self, session_id: str) -> Optional[VerificationSession]:
301
+ """
302
+ Attempt to recover a corrupted session.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
303
 
304
+ Args:
305
+ session_id: ID of session to recover
 
 
 
306
 
307
+ Returns:
308
+ Recovered session if possible, None otherwise
309
+ """
310
  try:
311
+ filename = f"{session_id}.json"
312
+ filepath = os.path.join(self.storage_dir, filename)
 
 
 
 
313
 
314
+ if not os.path.exists(filepath):
315
+ return None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
316
 
317
+ # Try to read raw file content
318
+ with open(filepath, 'r', encoding='utf-8') as f:
319
+ content = f.read()
320
 
321
+ # Attempt basic JSON repair (remove trailing commas, etc.)
322
+ content = content.replace(',}', '}').replace(',]', ']')
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
323
 
324
+ try:
325
+ session_dict = json.loads(content)
326
+ except json.JSONDecodeError:
327
+ # If still can't parse, try to extract what we can
328
+ print(f"Severe corruption in session {session_id}, attempting partial recovery")
329
+ return None
330
+
331
+ # Try to reconstruct session with minimal required fields
332
+ required_fields = ['session_id', 'conversation_session_id', 'patient_name', 'verifier_name', 'start_time']
333
+ for field in required_fields:
334
+ if field not in session_dict:
335
+ print(f"Missing required field {field} in corrupted session")
336
+ return None
337
+
338
+ # Set defaults for missing optional fields
339
+ session_dict.setdefault('end_time', None)
340
+ session_dict.setdefault('total_exchanges', 0)
341
+ session_dict.setdefault('verified_exchanges', 0)
342
+ session_dict.setdefault('verification_records', [])
343
+ session_dict.setdefault('is_complete', False)
344
+
345
+ # Try to load the recovered session
346
+ return self.load_session(session_id)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
347
 
348
+ except Exception as e:
349
+ print(f"Recovery failed for session {session_id}: {e}")
350
+ return None
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
src/interface/conversation_verification_ui.py ADDED
@@ -0,0 +1,472 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """
3
+ Conversation Verification UI Components.
4
+
5
+ Gradio-based interface for reviewing and verifying AI classifier decisions
6
+ on patient conversations.
7
+ """
8
+
9
+ import gradio as gr
10
+ from typing import Dict, List, Optional, Tuple, Any
11
+ from datetime import datetime
12
+
13
+ from src.core.conversation_verification import (
14
+ VerificationSession, VerificationRecord, VerificationFeedback,
15
+ ConversationVerificationManager
16
+ )
17
+
18
+
19
+ class VerificationInterface:
20
+ """Gradio-based UI for conversation verification."""
21
+
22
+ def __init__(self, manager: ConversationVerificationManager):
23
+ """Initialize verification interface."""
24
+ self.manager = manager
25
+ self.current_session: Optional[VerificationSession] = None
26
+ self.current_record_index: int = 0
27
+
28
+ def create_verification_window(self, session: VerificationSession) -> gr.Blocks:
29
+ """
30
+ Create main verification window interface.
31
+
32
+ Args:
33
+ session: VerificationSession to review
34
+
35
+ Returns:
36
+ Gradio Blocks interface
37
+ """
38
+ self.current_session = session
39
+ self.current_record_index = 0
40
+
41
+ with gr.Blocks(
42
+ title=f"Verify Conversation - {session.patient_name}",
43
+ theme=gr.themes.Soft()
44
+ ) as interface:
45
+
46
+ # Session header
47
+ with gr.Row():
48
+ gr.Markdown(f"""
49
+ # πŸ” Conversation Verification
50
+
51
+ **Patient:** {session.patient_name}
52
+ **Verifier:** {session.verifier_name}
53
+ **Session:** `{session.session_id}`
54
+ **Started:** {session.start_time.strftime('%Y-%m-%d %H:%M')}
55
+ """)
56
+
57
+ # Progress section
58
+ with gr.Row():
59
+ progress_bar = gr.HTML(value=self._render_progress_bar(session))
60
+
61
+ with gr.Row():
62
+ with gr.Column(scale=1):
63
+ stats_display = gr.HTML(value=self._render_statistics(session))
64
+ with gr.Column(scale=1):
65
+ navigation_info = gr.HTML(value=self._render_navigation_info(session))
66
+
67
+ # Main verification area
68
+ with gr.Row():
69
+ with gr.Column(scale=3):
70
+ # Current exchange display
71
+ exchange_display = gr.HTML(value="", label="Current Exchange")
72
+
73
+ # Verification buttons
74
+ with gr.Row():
75
+ correct_btn = gr.Button("βœ… Correct", variant="primary", scale=1)
76
+ incorrect_btn = gr.Button("❌ Incorrect", variant="secondary", scale=1)
77
+
78
+ # Correction interface (initially hidden)
79
+ correction_section = gr.Column(visible=False)
80
+ with correction_section:
81
+ gr.Markdown("### Select Correct Classification:")
82
+ correction_radio = gr.Radio(
83
+ choices=["GREEN", "YELLOW", "RED"],
84
+ label="Correct Classification",
85
+ interactive=True
86
+ )
87
+
88
+ correction_reason = gr.Dropdown(
89
+ choices=[
90
+ "Missed indicators",
91
+ "False positive",
92
+ "Context misunderstanding",
93
+ "Severity misjudgment",
94
+ "Other"
95
+ ],
96
+ label="Correction Reason",
97
+ interactive=True
98
+ )
99
+
100
+ correction_notes = gr.Textbox(
101
+ label="Additional Notes (Optional)",
102
+ placeholder="Explain the correction...",
103
+ lines=3,
104
+ interactive=True
105
+ )
106
+
107
+ with gr.Row():
108
+ submit_correction_btn = gr.Button("βœ… Submit Correction", variant="primary")
109
+ cancel_correction_btn = gr.Button("❌ Cancel", variant="secondary")
110
+
111
+ with gr.Column(scale=1):
112
+ # Navigation controls
113
+ with gr.Column():
114
+ gr.Markdown("### Navigation")
115
+
116
+ with gr.Row():
117
+ prev_btn = gr.Button("⬅️ Previous", scale=1)
118
+ next_btn = gr.Button("Next ➑️", scale=1)
119
+
120
+ current_position = gr.HTML(value="Exchange 1 of 1")
121
+
122
+ # Quick actions
123
+ gr.Markdown("### Quick Actions")
124
+ mark_all_correct_btn = gr.Button("βœ… Mark All Remaining as Correct", size="sm")
125
+
126
+ # Export section
127
+ gr.Markdown("### Export Results")
128
+ export_btn = gr.Button("πŸ“Š Export to CSV", variant="primary")
129
+ export_status = gr.HTML(value="")
130
+
131
+ # Hidden state components
132
+ session_state = gr.State(value=session)
133
+ current_index = gr.State(value=0)
134
+
135
+ # Load initial exchange
136
+ interface.load(
137
+ fn=self._load_initial_exchange,
138
+ inputs=[session_state, current_index],
139
+ outputs=[exchange_display, current_position, stats_display, progress_bar]
140
+ )
141
+
142
+ # Event handlers
143
+ correct_btn.click(
144
+ fn=self._handle_correct_feedback,
145
+ inputs=[session_state, current_index],
146
+ outputs=[exchange_display, current_position, stats_display, progress_bar, current_index]
147
+ )
148
+
149
+ incorrect_btn.click(
150
+ fn=lambda: gr.Column(visible=True),
151
+ outputs=[correction_section]
152
+ )
153
+
154
+ submit_correction_btn.click(
155
+ fn=self._handle_incorrect_feedback,
156
+ inputs=[session_state, current_index, correction_radio, correction_reason, correction_notes],
157
+ outputs=[exchange_display, current_position, stats_display, progress_bar, current_index, correction_section]
158
+ )
159
+
160
+ cancel_correction_btn.click(
161
+ fn=lambda: (gr.Column(visible=False), "", "", ""),
162
+ outputs=[correction_section, correction_radio, correction_reason, correction_notes]
163
+ )
164
+
165
+ prev_btn.click(
166
+ fn=self._navigate_previous,
167
+ inputs=[session_state, current_index],
168
+ outputs=[exchange_display, current_position, current_index]
169
+ )
170
+
171
+ next_btn.click(
172
+ fn=self._navigate_next,
173
+ inputs=[session_state, current_index],
174
+ outputs=[exchange_display, current_position, current_index]
175
+ )
176
+
177
+ mark_all_correct_btn.click(
178
+ fn=self._mark_all_remaining_correct,
179
+ inputs=[session_state, current_index],
180
+ outputs=[exchange_display, current_position, stats_display, progress_bar]
181
+ )
182
+
183
+ export_btn.click(
184
+ fn=self._export_results,
185
+ inputs=[session_state],
186
+ outputs=[export_status]
187
+ )
188
+
189
+ return interface
190
+
191
+ def _load_initial_exchange(self, session: VerificationSession, index: int) -> Tuple[str, str, str, str]:
192
+ """Load the first exchange for verification."""
193
+ if not session.verification_records:
194
+ return "No exchanges to verify", "No exchanges", "", ""
195
+
196
+ record = session.verification_records[index]
197
+ exchange_html = self._render_exchange_review(record)
198
+ position_html = f"Exchange {index + 1} of {len(session.verification_records)}"
199
+ stats_html = self._render_statistics(session)
200
+ progress_html = self._render_progress_bar(session)
201
+
202
+ return exchange_html, position_html, stats_html, progress_html
203
+
204
+ def _render_exchange_review(self, record: VerificationRecord) -> str:
205
+ """Render exchange for review."""
206
+ # Classification indicator
207
+ indicator_emoji = {"GREEN": "🟒", "YELLOW": "🟑", "RED": "πŸ”΄"}
208
+ emoji = indicator_emoji.get(record.original_classification, "βšͺ")
209
+
210
+ # Verification status
211
+ status_html = ""
212
+ if record.is_correct is not None:
213
+ if record.is_correct:
214
+ status_html = '<div style="background-color: #d4edda; padding: 0.5em; border-radius: 4px; margin-bottom: 1em;"><strong>βœ… Verified as Correct</strong></div>'
215
+ else:
216
+ status_html = f'''<div style="background-color: #f8d7da; padding: 0.5em; border-radius: 4px; margin-bottom: 1em;">
217
+ <strong>❌ Marked as Incorrect</strong><br>
218
+ Correct Classification: <strong>{record.correct_classification}</strong><br>
219
+ Reason: {record.correction_reason}<br>
220
+ {f"Notes: {record.verifier_notes}" if record.verifier_notes else ""}
221
+ </div>'''
222
+
223
+ # Indicators display
224
+ indicators_html = ""
225
+ if record.original_indicators:
226
+ indicators_list = ", ".join(record.original_indicators[:3])
227
+ if len(record.original_indicators) > 3:
228
+ indicators_list += f" +{len(record.original_indicators) - 3} more"
229
+ indicators_html = f"<br><em>Indicators: {indicators_list}</em>"
230
+
231
+ html = f"""
232
+ <div style="border: 1px solid #ddd; border-radius: 8px; padding: 1em; margin-bottom: 1em;">
233
+ {status_html}
234
+
235
+ <div style="background-color: #f8f9fa; padding: 0.75em; border-radius: 4px; margin-bottom: 1em;">
236
+ <strong>πŸ‘€ Patient Message:</strong><br>
237
+ <em>"{record.user_message}"</em>
238
+ </div>
239
+
240
+ <div style="background-color: #e3f2fd; padding: 0.75em; border-radius: 4px; margin-bottom: 1em;">
241
+ <strong>πŸ€– AI Response:</strong><br>
242
+ {record.assistant_response}
243
+ </div>
244
+
245
+ <div style="background-color: #fff3e0; padding: 0.75em; border-radius: 4px;">
246
+ <strong>πŸ” AI Classification:</strong><br>
247
+ {emoji} <strong>{record.original_classification}</strong> ({int(record.original_confidence * 100)}%)
248
+ {indicators_html}
249
+ <br><em>Reasoning: {record.original_reasoning}</em>
250
+ </div>
251
+ </div>
252
+ """
253
+
254
+ return html
255
+
256
+ def _render_progress_bar(self, session: VerificationSession) -> str:
257
+ """Render progress bar."""
258
+ progress = session.get_progress()
259
+ percentage = progress.calculate_progress_percentage()
260
+
261
+ return f"""
262
+ <div style="margin-bottom: 1em;">
263
+ <div style="display: flex; justify-content: space-between; margin-bottom: 0.5em;">
264
+ <span><strong>Progress:</strong> {progress.verified_exchanges} of {progress.total_exchanges} verified</span>
265
+ <span><strong>{percentage:.1f}%</strong></span>
266
+ </div>
267
+ <div style="background-color: #e9ecef; border-radius: 10px; height: 20px;">
268
+ <div style="background-color: #28a745; height: 20px; border-radius: 10px; width: {percentage}%;"></div>
269
+ </div>
270
+ </div>
271
+ """
272
+
273
+ def _render_statistics(self, session: VerificationSession) -> str:
274
+ """Render verification statistics."""
275
+ progress = session.get_progress()
276
+
277
+ if progress.verified_exchanges == 0:
278
+ return """
279
+ <div style="background-color: #f8f9fa; padding: 1em; border-radius: 8px;">
280
+ <h4>πŸ“Š Statistics</h4>
281
+ <p>No verifications completed yet.</p>
282
+ </div>
283
+ """
284
+
285
+ stats_html = f"""
286
+ <div style="background-color: #f8f9fa; padding: 1em; border-radius: 8px;">
287
+ <h4>πŸ“Š Statistics</h4>
288
+ <p><strong>Overall Accuracy:</strong> {progress.accuracy_overall:.1%}</p>
289
+
290
+ <p><strong>By Classification:</strong></p>
291
+ <ul>
292
+ <li>🟒 GREEN: {progress.accuracy_by_type.get('GREEN', 0):.1%}</li>
293
+ <li>🟑 YELLOW: {progress.accuracy_by_type.get('YELLOW', 0):.1%}</li>
294
+ <li>πŸ”΄ RED: {progress.accuracy_by_type.get('RED', 0):.1%}</li>
295
+ </ul>
296
+ """
297
+
298
+ if progress.common_errors:
299
+ stats_html += "<p><strong>Common Errors:</strong></p><ul>"
300
+ for from_class, to_class, count in progress.common_errors[:3]:
301
+ stats_html += f"<li>{from_class} β†’ {to_class}: {count} cases</li>"
302
+ stats_html += "</ul>"
303
+
304
+ stats_html += "</div>"
305
+ return stats_html
306
+
307
+ def _render_navigation_info(self, session: VerificationSession) -> str:
308
+ """Render navigation information."""
309
+ unverified_count = len(session.get_unverified_records())
310
+
311
+ return f"""
312
+ <div style="background-color: #e3f2fd; padding: 1em; border-radius: 8px;">
313
+ <h4>🧭 Navigation</h4>
314
+ <p><strong>Total Exchanges:</strong> {session.total_exchanges}</p>
315
+ <p><strong>Remaining:</strong> {unverified_count}</p>
316
+ <p><strong>Status:</strong> {'Complete' if session.is_complete else 'In Progress'}</p>
317
+ </div>
318
+ """
319
+
320
+ def _handle_correct_feedback(self, session: VerificationSession, index: int) -> Tuple[str, str, str, str, int]:
321
+ """Handle correct classification feedback."""
322
+ if index >= len(session.verification_records):
323
+ return "No more exchanges", f"Exchange {index + 1} of {len(session.verification_records)}", "", "", index
324
+
325
+ record = session.verification_records[index]
326
+
327
+ # Submit feedback
328
+ feedback = VerificationFeedback(
329
+ exchange_id=record.exchange_id,
330
+ is_correct=True
331
+ )
332
+
333
+ self.manager.submit_exchange_verification(session.session_id, record.exchange_id, feedback)
334
+
335
+ # Reload session to get updated data
336
+ updated_session = self.manager.load_session(session.session_id)
337
+
338
+ # Move to next unverified exchange
339
+ next_index = self._find_next_unverified_index(updated_session, index)
340
+
341
+ if next_index is not None:
342
+ next_record = updated_session.verification_records[next_index]
343
+ exchange_html = self._render_exchange_review(next_record)
344
+ position_html = f"Exchange {next_index + 1} of {len(updated_session.verification_records)}"
345
+ else:
346
+ exchange_html = "<div style='text-align: center; padding: 2em;'><h3>πŸŽ‰ All exchanges verified!</h3><p>You can now export the results.</p></div>"
347
+ position_html = "Verification Complete"
348
+ next_index = index
349
+
350
+ stats_html = self._render_statistics(updated_session)
351
+ progress_html = self._render_progress_bar(updated_session)
352
+
353
+ return exchange_html, position_html, stats_html, progress_html, next_index
354
+
355
+ def _handle_incorrect_feedback(
356
+ self,
357
+ session: VerificationSession,
358
+ index: int,
359
+ correct_classification: str,
360
+ correction_reason: str,
361
+ notes: str
362
+ ) -> Tuple[str, str, str, str, int, gr.Column]:
363
+ """Handle incorrect classification feedback."""
364
+ if index >= len(session.verification_records):
365
+ return "No more exchanges", f"Exchange {index + 1} of {len(session.verification_records)}", "", "", index, gr.Column(visible=False)
366
+
367
+ record = session.verification_records[index]
368
+
369
+ # Submit feedback
370
+ feedback = VerificationFeedback(
371
+ exchange_id=record.exchange_id,
372
+ is_correct=False,
373
+ correct_classification=correct_classification,
374
+ correction_reason=correction_reason,
375
+ notes=notes if notes.strip() else None
376
+ )
377
+
378
+ self.manager.submit_exchange_verification(session.session_id, record.exchange_id, feedback)
379
+
380
+ # Reload session
381
+ updated_session = self.manager.load_session(session.session_id)
382
+
383
+ # Move to next unverified exchange
384
+ next_index = self._find_next_unverified_index(updated_session, index)
385
+
386
+ if next_index is not None:
387
+ next_record = updated_session.verification_records[next_index]
388
+ exchange_html = self._render_exchange_review(next_record)
389
+ position_html = f"Exchange {next_index + 1} of {len(updated_session.verification_records)}"
390
+ else:
391
+ exchange_html = "<div style='text-align: center; padding: 2em;'><h3>πŸŽ‰ All exchanges verified!</h3><p>You can now export the results.</p></div>"
392
+ position_html = "Verification Complete"
393
+ next_index = index
394
+
395
+ stats_html = self._render_statistics(updated_session)
396
+ progress_html = self._render_progress_bar(updated_session)
397
+
398
+ return exchange_html, position_html, stats_html, progress_html, next_index, gr.Column(visible=False)
399
+
400
+ def _navigate_previous(self, session: VerificationSession, index: int) -> Tuple[str, str, int]:
401
+ """Navigate to previous exchange."""
402
+ new_index = max(0, index - 1)
403
+ record = session.verification_records[new_index]
404
+ exchange_html = self._render_exchange_review(record)
405
+ position_html = f"Exchange {new_index + 1} of {len(session.verification_records)}"
406
+
407
+ return exchange_html, position_html, new_index
408
+
409
+ def _navigate_next(self, session: VerificationSession, index: int) -> Tuple[str, str, int]:
410
+ """Navigate to next exchange."""
411
+ new_index = min(len(session.verification_records) - 1, index + 1)
412
+ record = session.verification_records[new_index]
413
+ exchange_html = self._render_exchange_review(record)
414
+ position_html = f"Exchange {new_index + 1} of {len(session.verification_records)}"
415
+
416
+ return exchange_html, position_html, new_index
417
+
418
+ def _mark_all_remaining_correct(self, session: VerificationSession, current_index: int) -> Tuple[str, str, str, str]:
419
+ """Mark all remaining unverified exchanges as correct."""
420
+ unverified_records = session.get_unverified_records()
421
+
422
+ for record in unverified_records:
423
+ feedback = VerificationFeedback(
424
+ exchange_id=record.exchange_id,
425
+ is_correct=True
426
+ )
427
+ self.manager.submit_exchange_verification(session.session_id, record.exchange_id, feedback)
428
+
429
+ # Reload session
430
+ updated_session = self.manager.load_session(session.session_id)
431
+
432
+ exchange_html = "<div style='text-align: center; padding: 2em;'><h3>πŸŽ‰ All exchanges verified!</h3><p>All remaining exchanges marked as correct.</p></div>"
433
+ position_html = "Verification Complete"
434
+ stats_html = self._render_statistics(updated_session)
435
+ progress_html = self._render_progress_bar(updated_session)
436
+
437
+ return exchange_html, position_html, stats_html, progress_html
438
+
439
+ def _export_results(self, session: VerificationSession) -> str:
440
+ """Export verification results to CSV."""
441
+ try:
442
+ from src.core.verification_exporter import VerificationExporter
443
+ exporter = VerificationExporter()
444
+ csv_path = exporter.export_session_to_csv(session)
445
+
446
+ return f"""
447
+ <div style="background-color: #d4edda; padding: 1em; border-radius: 4px;">
448
+ <strong>βœ… Export Successful!</strong><br>
449
+ File saved: <code>{csv_path}</code><br>
450
+ <small>Check your downloads folder</small>
451
+ </div>
452
+ """
453
+ except Exception as e:
454
+ return f"""
455
+ <div style="background-color: #f8d7da; padding: 1em; border-radius: 4px;">
456
+ <strong>❌ Export Failed</strong><br>
457
+ Error: {str(e)}
458
+ </div>
459
+ """
460
+
461
+ def _find_next_unverified_index(self, session: VerificationSession, current_index: int) -> Optional[int]:
462
+ """Find the next unverified exchange index."""
463
+ for i in range(current_index + 1, len(session.verification_records)):
464
+ if session.verification_records[i].is_correct is None:
465
+ return i
466
+
467
+ # If no unverified found after current, check from beginning
468
+ for i in range(current_index):
469
+ if session.verification_records[i].is_correct is None:
470
+ return i
471
+
472
+ return None # All verified
src/interface/simplified_gradio_app.py CHANGED
@@ -302,6 +302,10 @@ def create_simplified_interface():
302
  download_json_btn = gr.DownloadButton("πŸ“₯ Download JSON", scale=1, size="sm")
303
  download_csv_btn = gr.DownloadButton("πŸ“Š Download CSV", scale=1, size="sm")
304
 
 
 
 
 
305
  # Quick examples
306
  gr.Markdown("### ⚑ Quick Start:")
307
  with gr.Row():
@@ -778,6 +782,44 @@ Changes apply only to your current session.
778
  print(f"Error downloading CSV: {e}")
779
  return None
780
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
781
  # Prompt editing handlers
782
  def format_prompt_with_html(prompt_text: str) -> str:
783
  """Format prompt with HTML tags for better visualization."""
@@ -1877,6 +1919,13 @@ To revert, use "Reset to Default" button.
1877
  outputs=[download_csv_btn]
1878
  )
1879
 
 
 
 
 
 
 
 
1880
  # Refresh conversation stats
1881
  refresh_stats_btn.click(
1882
  get_conversation_stats,
 
302
  download_json_btn = gr.DownloadButton("πŸ“₯ Download JSON", scale=1, size="sm")
303
  download_csv_btn = gr.DownloadButton("πŸ“Š Download CSV", scale=1, size="sm")
304
 
305
+ # Verification section
306
+ with gr.Row():
307
+ verify_conversation_btn = gr.Button("πŸ” Verify Conversation", variant="secondary", scale=2, size="sm")
308
+
309
  # Quick examples
310
  gr.Markdown("### ⚑ Quick Start:")
311
  with gr.Row():
 
782
  print(f"Error downloading CSV: {e}")
783
  return None
784
 
785
+ def open_verification_window(session: SimplifiedSessionData):
786
+ """Open verification window for current conversation."""
787
+ if session is None or not hasattr(session.app_instance, 'conversation_logger'):
788
+ return "❌ No conversation to verify"
789
+
790
+ try:
791
+ # Check if conversation has any entries
792
+ if not session.app_instance.conversation_logger.entries:
793
+ return "❌ No conversation exchanges to verify"
794
+
795
+ # Create verification session
796
+ from src.core.conversation_verification import ConversationVerificationManager
797
+ from src.interface.conversation_verification_ui import VerificationInterface
798
+
799
+ manager = ConversationVerificationManager()
800
+ verification_session = manager.create_verification_session(
801
+ session.app_instance.conversation_logger,
802
+ "Medical Professional"
803
+ )
804
+
805
+ # Create verification interface
806
+ interface = VerificationInterface(manager)
807
+ verification_window = interface.create_verification_window(verification_session)
808
+
809
+ # Launch verification window
810
+ verification_window.launch(
811
+ server_name="127.0.0.1",
812
+ server_port=7861, # Different port from main app
813
+ share=False,
814
+ show_error=True,
815
+ quiet=True
816
+ )
817
+
818
+ return f"βœ… Verification window opened for {len(verification_session.verification_records)} exchanges"
819
+
820
+ except Exception as e:
821
+ return f"❌ Error opening verification: {str(e)}"
822
+
823
  # Prompt editing handlers
824
  def format_prompt_with_html(prompt_text: str) -> str:
825
  """Format prompt with HTML tags for better visualization."""
 
1919
  outputs=[download_csv_btn]
1920
  )
1921
 
1922
+ # Verification button
1923
+ verify_conversation_btn.click(
1924
+ open_verification_window,
1925
+ inputs=[session_data],
1926
+ outputs=[] # No outputs needed as it opens new window
1927
+ )
1928
+
1929
  # Refresh conversation stats
1930
  refresh_stats_btn.click(
1931
  get_conversation_stats,