add extra columns for feedback functionality

src/reporting/feedback_schema.py

@@ -4,10 +4,12 @@ Feedback Schema for RAG Chatbot
 This module defines dataclasses for feedback data structures
 and provides Snowflake schema generation.
 """
-
+import os
+from datetime import datetime
 from dataclasses import dataclass, asdict, field
 from typing import List, Optional, Dict, Any, Union
-from datetime import datetime
+
+
 
 
 @dataclass
@@ -39,34 +41,20 @@ class UserFeedback:
     open_ended_feedback: Optional[str]
     score: int
     is_feedback_about_last_retrieval: bool
-    retrieved_data: List[RetrievalEntry]
     conversation_id: str
     timestamp: float
     message_count: int
     has_retrievals: bool
     retrieval_count: int
-    user_query: Optional[str] = None
-    bot_response: Optional[str] = None
+    transcript: List[Dict[str, str]]  # List of {"role": "user"/"assistant", "content": "..."}
+    retrievals: List[Dict[str, Any]]  # List of retrieval objects with retrieved_docs and user_message_trigger
+    feedback_score_related_retrieval_docs: Optional[Dict[str, Any]] = None  # Conversation subset + retrieved docs
+    retrieved_data: Optional[List[Dict[str, Any]]] = None  # Preserved old column for backward compatibility
     created_at: str = field(default_factory=lambda: datetime.now().isoformat())
     
     def to_dict(self) -> Dict[str, Any]:
         """Convert to dictionary with nested data structures"""
         result = asdict(self)
-        # Handle nested objects
-        if self.retrieved_data:
-            result['retrieved_data'] = [self._serialize_retrieval_entry(entry) for entry in self.retrieved_data]
-        return result
-    
-    def _serialize_retrieval_entry(self, entry: RetrievalEntry) -> Dict[str, Any]:
-        """Serialize retrieval entry to dict"""
-        # If raw data exists, use it (it's already properly formatted)
-        if hasattr(entry, '_raw_data') and entry._raw_data:
-            return entry._raw_data
-        
-        # Otherwise, serialize the dataclass
-        result = asdict(entry)
-        if entry.documents_retrieved:
-            result['documents_retrieved'] = [asdict(doc) for doc in entry.documents_retrieved]
         return result
     
     def to_snowflake_schema(self) -> Dict[str, Any]:
@@ -81,28 +69,28 @@ class UserFeedback:
             "message_count": "INTEGER",
             "has_retrievals": "BOOLEAN",
             "retrieval_count": "INTEGER",
-            "user_query": "VARCHAR(16777216)",
-            "bot_response": "VARCHAR(16777216)",
+            "transcript": "VARCHAR(16777216)",  # JSON string of ARRAY of {"role": "user"/"assistant", "content": "..."}
+            "retrievals": "VARCHAR(16777216)",  # JSON string of ARRAY of retrieval objects
+            "feedback_score_related_retrieval_docs": "VARCHAR(16777216)",  # JSON string of OBJECT with conversation subset + retrieved docs
+            "retrieved_data": "VARCHAR(16777216)",  # JSON string - preserved old column for backward compatibility
             "created_at": "TIMESTAMP_NTZ",
-            "retrieved_data": "VARIANT",  # Array of retrieval entries
-            # retrieved_data structure:
-            # [
+            # transcript structure: [{"role": "user", "content": "..."}, {"role": "assistant", "content": "..."}, ...]
+            # retrievals structure: [
             #   {
-            #     "rag_query": "...",
-            #     "conversation_length": 5,
-            #     "timestamp": 1234567890,
-            #     "docs_retrieved": [
-            #       {"filename": "...", "page": 14, "score": 0.95, ...},
-            #       ...
-            #     ]
+            #     "retrieved_docs": [{"content": "...", "metadata": {...}, ...}],  # content truncated to 100 chars
+            #     "user_message_trigger": "final user message that triggered this retrieval"
             #   },
             #   ...
             # ]
+            # feedback_score_related_retrieval_docs structure: {
+            #   "conversation_up_to_point": [{"role": "user", "content": "..."}, ...],  # subset of transcript
+            #   "retrieved_docs": [{"content": "...", "metadata": {...}, ...}]  # full chunks with all info
+            # }
         }
         return schema
     
     @classmethod
-    def get_snowflake_create_table_sql(cls, table_name: str = "user_feedback") -> str:
+    def get_snowflake_create_table_sql(cls, table_name: str = "USER_FEEDBACK_V3") -> str:
         """Generate CREATE TABLE SQL for Snowflake"""
         schema = cls.to_snowflake_schema(None)
         
@@ -117,16 +105,13 @@ class UserFeedback:
         sql = f"""CREATE TABLE IF NOT EXISTS {table_name} (
 {columns_str},
   PRIMARY KEY (feedback_id)
-);
-
--- Create index on timestamp for querying by time
-CREATE INDEX IF NOT EXISTS idx_feedback_timestamp ON {table_name} (timestamp);
-
--- Create index on conversation_id for querying by conversation
-CREATE INDEX IF NOT EXISTS idx_feedback_conversation ON {table_name} (conversation_id);
-
--- Create index on score for feedback analysis
-CREATE INDEX IF NOT EXISTS idx_feedback_score ON {table_name} (score);
+)
+CLUSTER BY (timestamp, conversation_id, score);
+-- Note: Snowflake doesn't support traditional indexes on regular tables.
+-- Instead, we use CLUSTER BY to optimize queries on these columns.
+-- Snowflake automatically maintains clustering for efficient querying.
+-- Note: transcript, retrievals, and feedback_score_related_retrieval_docs are stored as VARCHAR (JSON strings),
+-- same approach as the old retrieved_data column. This allows easy storage and retrieval without VARIANT type complexity.
 """
         return sql
 
@@ -150,47 +135,27 @@ DOCUMENT_SCHEMA = {
 }
 
 
-def generate_snowflake_schema_sql() -> str:
+def generate_snowflake_schema_sql(table_name: Optional[str] = None) -> str:
     """Generate complete Snowflake schema SQL for feedback system"""
-    return UserFeedback.get_snowflake_create_table_sql("user_feedback")
+    if table_name is None:
+        table_name = os.getenv("SNOWFLAKE_FEEDBACK_TABLE", "USER_FEEDBACK_V3")
+    return UserFeedback.get_snowflake_create_table_sql(table_name)
 
 
 def create_feedback_from_dict(data: Dict[str, Any]) -> UserFeedback:
     """Create UserFeedback instance from dictionary"""
-    # Parse retrieved_data if present
-    retrieved_data = []
-    if "retrieved_data" in data and data["retrieved_data"]:
-        for entry_dict in data.get("retrieved_data", []):
-            # Map the actual structure from rag_retrieval_history
-            # Entry has: conversation_up_to, rag_query_expansion, docs_retrieved
-            try:
-                # Try to map to expected structure
-                entry = RetrievalEntry(
-                    rag_query=entry_dict.get("rag_query_expansion", ""),
-                    documents_retrieved=[],  # Empty for now, will store as raw data
-                    conversation_length=len(entry_dict.get("conversation_up_to", [])),
-                    filters_applied=None,
-                    timestamp=entry_dict.get("timestamp", None)
-                )
-                # Store raw data in the entry
-                entry._raw_data = entry_dict  # Store original for preservation
-                retrieved_data.append(entry)
-            except Exception as e:
-                # If mapping fails, store as-is without strict typing
-                pass
-    
     return UserFeedback(
         feedback_id=data.get("feedback_id", f"feedback_{data.get('timestamp', 'unknown')}"),
         open_ended_feedback=data.get("open_ended_feedback"),
         score=data["score"],
         is_feedback_about_last_retrieval=data["is_feedback_about_last_retrieval"],
-        retrieved_data=retrieved_data,
         conversation_id=data["conversation_id"],
         timestamp=data["timestamp"],
         message_count=data["message_count"],
         has_retrievals=data["has_retrievals"],
         retrieval_count=data["retrieval_count"],
-        user_query=data.get("user_query"),
-        bot_response=data.get("bot_response")
+        transcript=data.get("transcript", []),
+        retrievals=data.get("retrievals", []),
+        feedback_score_related_retrieval_docs=data.get("feedback_score_related_retrieval_docs"),
+        retrieved_data=data.get("retrieved_data")
     )
-