v1.1

requirements.txt

@@ -1,8 +1,11 @@
-gradio==4.44.0
+gradio==4.44.1
 openai>=1.0.0
 elevenlabs==1.10.0
 llama-index==0.11.20
 llama-index-llms-openai>=0.1.5
+llama-index-embeddings-openai>=0.1.5
+llama-index-embeddings-huggingface>=0.1.0
+sentence-transformers>=2.2.0
 requests==2.32.3
 python-dotenv==1.0.1
 pydantic==2.9.2

src/app.py

@@ -80,12 +80,15 @@ def save_preferences(name: str, favorite_genres: List[str], interests: List[str]
     radio_state["content_filter"] = preferences["content_filter"]
     
     radio_state["user_preferences"] = preferences
-    agent.rag_system.store_user_preferences(preferences)
+    
+    # Save to RAG system with user_id
+    user_id = radio_state.get("user_id")
+    agent.rag_system.store_user_preferences(preferences, user_id=user_id)
     
     # Save to user memory for persistence
-    if radio_state.get("user_id"):
-        user_memory.save_user_preferences(radio_state["user_id"], preferences)
-        print(f"💾 Preferences saved for user {radio_state['user_id']}")
+    if user_id:
+        user_memory.save_user_preferences(user_id, preferences)
+        print(f"💾 Preferences saved for user {user_id}")
     
     return f"✅ Preferences saved! Welcome, {preferences['name']}! Your personalized radio is ready."
 
@@ -151,7 +154,9 @@ def start_radio_stream():
         return "📻 Radio is already playing!", None, None, None, ""
     
     # Plan the show with content filter
+    user_id = radio_state.get("user_id")
     show_plan = agent.plan_radio_show(
+        user_id=user_id,
         user_preferences=radio_state["user_preferences"],
         duration_minutes=30,
         content_filter=radio_state["content_filter"]
@@ -227,7 +232,9 @@ def start_and_play_first_segment():
     
     # Step 1: Plan skeleton show (YouTube searches happen here)
     print("  [1/3] Planning show (searching YouTube)...")
+    user_id = radio_state.get("user_id")
     show_plan = agent.plan_radio_show(
+        user_id=user_id,
         user_preferences=radio_state["user_preferences"],
         duration_minutes=30,
         content_filter=radio_state["content_filter"]
@@ -315,11 +322,15 @@ def play_next_segment():
     radio_state["current_segment_index"] += 1
     
     # Lazily generate LLM content for this segment
-    segment = agent.enrich_segment(segment, radio_state["user_preferences"])
+    user_prefs = radio_state["user_preferences"].copy()
+    user_id = radio_state.get("user_id")
+    if user_id:
+        user_prefs["_user_id"] = user_id  # Pass user_id in preferences for RAG queries
+    segment = agent.enrich_segment(segment, user_prefs)
     radio_state["planned_show"][segment_index] = segment
     
     # Execute segment (log to RAG)
-    agent.execute_segment(segment)
+    agent.execute_segment(segment, user_id=user_id)
     radio_state["last_segment"] = segment
     
     # Generate content display and underlying script text
@@ -404,7 +415,7 @@ def play_next_segment():
                         youtube_id = track["url"].split("v=")[-1].split("&")[0]
                     
                     if youtube_id:
-                        # YouTube embed with direct iframe (Gradio sanitizes script tags)
+                        # Simple YouTube iframe embed with autoplay
                         music_player_html = f"""
                         <div style="padding: 1rem; background: linear-gradient(135deg, #1a1a2e 0%, #16213e 100%); border-radius: 12px; margin: 1rem 0; box-shadow: 0 4px 15px rgba(0,0,0,0.3);">
                             <h4 style="margin: 0 0 0.75rem 0; color: #fff; font-size: 1.1em;">🎵 {track.get('title', 'Unknown')}</h4>
@@ -1197,14 +1208,16 @@ with gr.Blocks(css=custom_css, title="AI Radio 🎵", theme=gr.themes.Soft()) as
                 dislike_btn = gr.Button("👎 Don't Play Again", variant="secondary", size="sm")
                 like_status = gr.Textbox(label="Rating Status", value="Rate the current song!", interactive=False, scale=2)
             
-            progress_text = gr.Textbox(label="Progress", value="Ready to start", interactive=False)
+            progress_text = gr.Textbox(label="Progress", value="Ready to start", interactive=False, visible=False)
             
             segment_info = gr.Markdown("**Welcome!** Set your preferences and start the radio.", elem_id="segment-display")
+            
             llm_script = gr.Textbox(
                 label="🧠 Model Script (LLM answer)",
                 value="Model-generated script will appear here for each segment.",
                 interactive=False,
-                lines=6
+                lines=6,
+                visible=False
             )
             
             gr.Markdown("**💡 Tip:** Host speech plays first, then music/podcasts will stream automatically!")
@@ -1212,7 +1225,7 @@ with gr.Blocks(css=custom_css, title="AI Radio 🎵", theme=gr.themes.Soft()) as
             audio_output = gr.Audio(label="🔊 Host Speech", autoplay=True, type="filepath", elem_id="host_audio")
             music_player = gr.HTML(label="🎵 Music/Podcast Player (streaming)")
             
-            status_text = gr.Textbox(label="Status", value="Ready", interactive=False)
+            status_text = gr.Textbox(label="Status", value="Ready", interactive=False, visible=False)
             
             # Connect buttons
             start_btn.click(

src/radio_agent.py

@@ -61,9 +61,10 @@ class RadioAgent:
         self.is_streaming = False
         self.current_segment = None
         self.segment_history = []
+        self.current_user_id = None  # Track current user for RAG queries
         
     def plan_radio_show(self, user_preferences: Dict[str, Any], duration_minutes: int = 30, 
-                       content_filter: Dict[str, bool] = None) -> List[Dict[str, Any]]:
+                       content_filter: Dict[str, bool] = None, user_id: str = None) -> List[Dict[str, Any]]:
         """
         Plan a personalized radio show based on user preferences
         This demonstrates autonomous planning behavior
@@ -72,10 +73,14 @@ class RadioAgent:
             user_preferences: User's preferences and mood
             duration_minutes: Total duration of the show
             content_filter: Dictionary with content type filters (music, news, podcasts, stories)
+            user_id: User ID for RAG queries
             
         Returns:
             List of planned segments
         """
+        # Store user_id for later use
+        self.current_user_id = user_id
+        
         segments = []
         
         # Default filter - all enabled
@@ -87,8 +92,8 @@ class RadioAgent:
                 "stories": True
             }
         
-        # Get user preferences from RAG
-        stored_prefs = self.rag_system.get_user_preferences()
+        # Get user preferences from RAG (filtered by user_id)
+        stored_prefs = self.rag_system.get_user_preferences(user_id=user_id)
         merged_prefs = {**stored_prefs, **user_preferences}
         
         # Calculate segment distribution based on filter
@@ -320,7 +325,7 @@ class RadioAgent:
             'duration': 2
         }
     
-    def _generate_music_commentary(self, track: Dict[str, Any], preferences: Dict[str, Any]) -> str:
+    def _generate_music_commentary(self, track: Dict[str, Any], preferences: Dict[str, Any], rag_context: List[Dict[str, Any]] = None) -> str:
         """Generate longer, more engaging commentary for music track with jokes/facts"""
         if not track or not self.client:
             return f"Here's a great track for you!"
@@ -328,12 +333,31 @@ class RadioAgent:
         mood = preferences.get('mood', 'happy')
         interests = preferences.get('interests', [])
         
+        # Add RAG context to prompt if available
+        rag_context_text = ""
+        if rag_context:
+            rag_context_text = "\n\nUser's listening history context (use this to personalize your commentary):\n"
+            for i, ctx in enumerate(rag_context[:2]):  # Use top 2 most relevant
+                ctx_text = ctx.get('text', '')[:200]
+                score = ctx.get('score', 0)
+                rag_context_text += f"- Context #{i+1} (relevance: {score:.4f}): {ctx_text}\n"
+            
+            llm_logger.info(
+                "RAG_CONTEXT | track=%s | context_docs=%d | context_preview=%s",
+                track.get('title', 'Unknown'),
+                len(rag_context),
+                rag_context_text[:300]
+            )
+            print(f"  📝 [RAG] Added {len(rag_context)} context documents to LLM prompt")
+        else:
+            print(f"  ℹ️  [RAG] No context available - using standard prompt")
+        
         try:
             prompt = f"""You are an energetic, entertaining radio DJ named Lera. This is a middle of a show. Introduce this song in a fun, engaging way:
             Title: {track['title']}
             Artist: {track['artist']}
             Genre: {track['genre']}
-            Listener mood: {mood}
+            Listener mood: {mood}{rag_context_text}
             
             Make it:
             - About 10-15 seconds of speech, max_tokens = 400
@@ -341,7 +365,11 @@ class RadioAgent:
             - Add a light joke or witty comment
             - Be enthusiastic and engaging
             - Sound natural, like a real radio host
-            - Connect it to the listener's mood if possible"""
+            - Connect it to the listener's mood if possible
+            - Reference their listening history if relevant (from context above)"""
+            
+            if rag_context:
+                print(f"  ✅ [RAG] LLM prompt includes RAG context - model will use personalized history")
             
             response = self.client.chat.completions.create(
                 model=self.config.nebius_model,
@@ -350,10 +378,31 @@ class RadioAgent:
                 max_tokens=400 #400
             )
             text = response.choices[0].message.content.strip()
+            
+            # Check if response references RAG context
+            rag_used = False
+            if rag_context:
+                # Check if response mentions anything from context
+                text_lower = text.lower()
+                for ctx in rag_context:
+                    ctx_text = ctx.get('text', '').lower()
+                    # Look for key terms from context in response
+                    key_terms = ['listened', 'enjoyed', 'prefer', 'favorite', 'history', 'before', 'previous', 'similar']
+                    if any(term in ctx_text and term in text_lower for term in key_terms):
+                        rag_used = True
+                        break
+                
+                if rag_used:
+                    print(f"  ✅ [RAG] LLM response appears to reference RAG context!")
+                    rag_logger.info(f"✅ RAG CONTEXT USED in commentary for {track.get('title')}")
+                else:
+                    print(f"  ℹ️  [RAG] LLM response doesn't explicitly reference context (may still be influenced)")
+            
             llm_logger.info(
-                "MUSIC | model=%s | prompt=%r | response=%r",
+                "MUSIC | model=%s | rag_used=%s | prompt_length=%d | response=%r",
                 self.config.nebius_model,
-                prompt,
+                rag_used,
+                len(prompt),
                 text[:1000],
             )
             return text
@@ -528,15 +577,20 @@ Don't use emojis. Just speak naturally like a real DJ."""
         else:
             return "evening"
     
-    def execute_segment(self, segment: Dict[str, Any]) -> Dict[str, Any]:
+    def execute_segment(self, segment: Dict[str, Any], user_id: str = None) -> Dict[str, Any]:
         """Execute a planned segment and log to RAG"""
         self.current_segment = segment
         self.segment_history.append(segment)
         
-        # Store in RAG system
+        # Use stored user_id if not provided
+        if not user_id:
+            user_id = self.current_user_id
+        
+        # Store in RAG system with user_id
         self.rag_system.store_listening_history(
             item_type=segment['type'],
-            item_data=segment
+            item_data=segment,
+            user_id=user_id
         )
         
         return segment
@@ -567,10 +621,42 @@ Don't use emojis. Just speak naturally like a real DJ."""
         elif seg_type == "music":
             # Lazy fetch track if not yet fetched
             if not segment.get("track"):
+                # Use RAG to get personalized recommendations
                 genres = prefs.get('favorite_genres', ['pop'])
                 mood = prefs.get('mood', 'happy')
+                
+                # Use stored user_id if available
+                user_id = prefs.get('_user_id') or self.current_user_id
+                
+                # Query RAG for music preferences based on listening history
+                rag_query = f"What music genres and styles does the user prefer? Favorite genres: {', '.join(genres)}, Mood: {mood}"
+                print(f"\n🎵 [RAG] Querying for genre recommendations for user {user_id}...")
+                rag_context = self.rag_system.query_user_context(rag_query, user_id=user_id, top_k=3)
+                
+                original_genres = genres.copy()
+                # Extract genre suggestions from RAG if available
+                if rag_context:
+                    print(f"🎵 [RAG] Analyzing {len(rag_context)} context documents for genre suggestions...")
+                    for i, ctx in enumerate(rag_context):
+                        ctx_text = ctx.get('text', '').lower()
+                        score = ctx.get('score', 0)
+                        # Look for genre mentions in context
+                        for genre in ['pop', 'rock', 'jazz', 'classical', 'electronic', 'hip-hop', 'country', 'indie', 'rap', 'blues', 'folk', 'metal', 'reggae', 'soul', 'r&b']:
+                            if genre in ctx_text and genre not in genres:
+                                genres.append(genre)
+                                print(f"  ✅ [RAG] Suggested genre from context #{i+1} (score: {score:.4f}): {genre}")
+                else:
+                    print(f"  ℹ️  [RAG] No context found, using original genres: {original_genres}")
+                
+                # Use RAG-enhanced genre selection
+                selected_genre = random.choice(genres) if genres else 'pop'
+                if selected_genre not in original_genres:
+                    print(f"🎵 [RAG] Using RAG-suggested genre: {selected_genre} (was: {original_genres})")
+                else:
+                    print(f"🎵 [RAG] Using genre: {selected_genre} (from preferences)")
+                
                 tracks = self.music_server.search_free_music(
-                    genre=random.choice(genres),
+                    genre=selected_genre,
                     mood=mood,
                     limit=1
                 )
@@ -578,7 +664,24 @@ Don't use emojis. Just speak naturally like a real DJ."""
             
             track = segment.get("track")
             if track and not segment.get("commentary"):
-                segment["commentary"] = self._generate_music_commentary(track, prefs)
+                # Use stored user_id if available
+                user_id = prefs.get('_user_id') or self.current_user_id
+                
+                # Use RAG context for more personalized commentary
+                rag_query = f"User's music listening history and preferences for {track.get('title', 'this track')} by {track.get('artist', 'this artist')}"
+                print(f"\n🎤 [RAG] Querying for commentary context for user {user_id}: {track.get('title')} by {track.get('artist')}")
+                rag_context = self.rag_system.query_user_context(rag_query, user_id=user_id, top_k=2)
+                
+                if rag_context:
+                    print(f"🎤 [RAG] Using {len(rag_context)} context documents for personalized commentary")
+                    for i, ctx in enumerate(rag_context):
+                        score = ctx.get('score', 0)
+                        preview = ctx.get('text', '')[:80].replace('\n', ' ')
+                        print(f"  📄 Context #{i+1} (score: {score:.4f}): {preview}...")
+                else:
+                    print(f"  ℹ️  [RAG] No context found, generating standard commentary")
+                
+                segment["commentary"] = self._generate_music_commentary(track, prefs, rag_context=rag_context)
 
         elif seg_type == "news":
             # Lazy fetch news if not yet fetched

src/rag_system.py

@@ -1,15 +1,29 @@
 """RAG System for User Preferences and History using LlamaIndex"""
 import json
 import os
+import logging
 from typing import Dict, Any, List
 from datetime import datetime
 from llama_index.core import VectorStoreIndex, Document, Settings
 from llama_index.core.storage.storage_context import StorageContext
 from llama_index.core.vector_stores import SimpleVectorStore
+from llama_index.core.node_parser import SimpleNodeParser
 from llama_index.llms.openai import OpenAI as LlamaOpenAI
+from llama_index.embeddings.openai import OpenAIEmbedding
 
 # Get project root directory (parent of src/)
 PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+LOG_DIR = os.path.join(PROJECT_ROOT, "logs")
+os.makedirs(LOG_DIR, exist_ok=True)
+
+# Setup RAG logger
+rag_logger = logging.getLogger("ai_radio.rag")
+if not rag_logger.handlers:
+    rag_logger.setLevel(logging.INFO)
+    _fh = logging.FileHandler(os.path.join(LOG_DIR, "rag_system.log"), encoding="utf-8")
+    _fmt = logging.Formatter("%(asctime)s [RAG] %(levelname)s: %(message)s")
+    _fh.setFormatter(_fmt)
+    rag_logger.addHandler(_fh)
 
 class RadioRAGSystem:
     """RAG system for storing and retrieving user preferences and listening history"""
@@ -32,19 +46,88 @@ class RadioRAGSystem:
                     temperature=0.7
                 )
                 self.llm_available = True
+                print("✅ RAG LLM initialized (Nebius GPT-OSS-120B)")
             except Exception as e:
                 print(f"Warning: Could not initialize Nebius/OpenAI LLM: {e}")
                 print("RAG system will work in fallback mode without LLM features")
                 self.llm_available = False
             
-            # For embeddings, we'll use a simple approach or skip for now
-            # Embeddings can be added later if needed
-            self.embedding_available = False  # Disabled for now - can add OpenAI embeddings later
+            # Enable embeddings - try local model first, then OpenAI
+            # Note: Nebius doesn't support embeddings, so we use local or OpenAI
+            self.embedding_available = False
+            try:
+                # First, try local sentence-transformers (no API key needed)
+                try:
+                    from llama_index.embeddings.huggingface import HuggingFaceEmbedding
+                    Settings.embed_model = HuggingFaceEmbedding(
+                        model_name="sentence-transformers/all-MiniLM-L6-v2"
+                    )
+                    self.embedding_available = True
+                    print("✅ RAG Embeddings enabled (local sentence-transformers/all-MiniLM-L6-v2)")
+                    rag_logger.info("✅ Using local HuggingFace embeddings: sentence-transformers/all-MiniLM-L6-v2")
+                except ImportError:
+                    # Fallback: Try OpenAI embeddings endpoint (requires OpenAI API key)
+                    openai_key = os.environ.get("OPENAI_API_KEY")
+                    if openai_key:
+                        try:
+                            Settings.embed_model = OpenAIEmbedding(
+                                api_key=openai_key,
+                                api_base="https://api.openai.com/v1",
+                                model="text-embedding-3-small"
+                            )
+                            self.embedding_available = True
+                            print("✅ RAG Embeddings enabled (OpenAI text-embedding-3-small via OPENAI_API_KEY)")
+                            rag_logger.info("✅ Using OpenAI embeddings: text-embedding-3-small")
+                        except Exception as e:
+                            print(f"⚠️  OpenAI embeddings failed: {e}")
+                            print("ℹ️  Embeddings disabled. RAG will use fallback mode.")
+                            rag_logger.warning(f"⚠️  OpenAI embeddings failed: {e}")
+                            self.embedding_available = False
+                    else:
+                        print("ℹ️  Embeddings disabled: No local model or OPENAI_API_KEY found.")
+                        print("   Install: pip install sentence-transformers")
+                        print("   Or set OPENAI_API_KEY environment variable.")
+                        rag_logger.warning("⚠️  No embeddings available - install sentence-transformers or set OPENAI_API_KEY")
+                        self.embedding_available = False
+                except Exception as e:
+                    print(f"⚠️  Local embeddings failed: {e}")
+                    # Try OpenAI as last resort
+                    openai_key = os.environ.get("OPENAI_API_KEY")
+                    if openai_key:
+                        try:
+                            Settings.embed_model = OpenAIEmbedding(
+                                api_key=openai_key,
+                                api_base="https://api.openai.com/v1",
+                                model="text-embedding-ada-002"  # Try older model
+                            )
+                            self.embedding_available = True
+                            print("✅ RAG Embeddings enabled (OpenAI text-embedding-ada-002)")
+                            rag_logger.info("✅ Using OpenAI embeddings: text-embedding-ada-002")
+                        except:
+                            print("ℹ️  All embedding options failed. RAG will use fallback mode.")
+                            rag_logger.warning("⚠️  All embedding options failed")
+                            self.embedding_available = False
+                    else:
+                        print("ℹ️  Embeddings disabled. RAG will use fallback mode.")
+                        rag_logger.warning("⚠️  Embeddings disabled - no fallback available")
+                        self.embedding_available = False
+            except Exception as e:
+                print(f"Warning: Could not initialize embeddings: {e}")
+                print("RAG will work without vector search (fallback mode)")
+                rag_logger.error(f"❌ Embedding initialization error: {e}")
+                self.embedding_available = False
         
         # Initialize vector store
         self.vector_store = SimpleVectorStore()
         self.storage_context = StorageContext.from_defaults(vector_store=self.vector_store)
         
+        # Configure chunk size to handle larger metadata
+        # Increase chunk size to 4096 to accommodate metadata (metadata can be up to 1438 chars)
+        Settings.chunk_size = 4096
+        Settings.chunk_overlap = 400
+        rag_logger.info(f"📏 RAG chunk size set to {Settings.chunk_size} (overlap: {Settings.chunk_overlap})")
+        print(f"📏 [RAG] Chunk size: {Settings.chunk_size}, Overlap: {Settings.chunk_overlap}")
+        
         # Load existing index or create new one
         self.index = None
         self.documents = []
@@ -53,44 +136,188 @@ class RadioRAGSystem:
         self._load_user_data()
     
     def _load_user_data(self):
-        """Load user data from file"""
+        """Load user data from file and build vector index"""
         if os.path.exists(self.user_data_file):
             try:
                 with open(self.user_data_file, 'r') as f:
                     data = json.load(f)
-                    self.documents = [Document(text=json.dumps(d)) for d in data]
-                    if self.documents and self.embedding_available:
-                        try:
-                            self.index = VectorStoreIndex.from_documents(
-                                self.documents,
-                                storage_context=self.storage_context
+                    
+                # Convert old format (JSON in text) to new format (descriptive text + raw_data in metadata)
+                self.documents = []
+                for d in data:
+                    try:
+                        # Create descriptive text for RAG retrieval
+                        if d.get("type") == "preferences":
+                            prefs = d.get("data", {})
+                            pref_text = f"""User Preferences:
+Name: {prefs.get('name', 'Unknown')}
+Favorite Genres: {', '.join(prefs.get('favorite_genres', []))}
+Interests: {', '.join(prefs.get('interests', []))}
+Podcast Interests: {', '.join(prefs.get('podcast_interests', []))}
+Mood: {prefs.get('mood', 'neutral')}
+Content Filter: Music={prefs.get('content_filter', {}).get('music', True)}, News={prefs.get('content_filter', {}).get('news', True)}, Podcasts={prefs.get('content_filter', {}).get('podcasts', True)}, Stories={prefs.get('content_filter', {}).get('stories', True)}
+"""
+                            doc = Document(
+                                text=pref_text,
+                                metadata={
+                                    "type": "preferences",
+                                    "user_id": d.get("user_id"),
+                                    "timestamp": d.get("timestamp", datetime.now().isoformat()),
+                                    # Removed raw_data to reduce metadata size - essential fields stored separately
+                                }
                             )
-                        except Exception as e:
-                            print(f"Warning: Could not build vector index: {e}")
-                            self.index = None
+                        elif d.get("type") == "history":
+                            item_type = d.get("item_type", "")
+                            item_data = d.get("data", {})
+                            user_feedback = d.get("feedback")
+                            
+                            if item_type == "music":
+                                track = item_data.get("track", {})
+                                history_text = f"""Music Listening History:
+Title: {track.get('title', 'Unknown')}
+Artist: {track.get('artist', 'Unknown')}
+Genre: {track.get('genre', 'Unknown')}
+Source: {track.get('source', 'Unknown')}
+Feedback: {user_feedback or 'No feedback'}
+"""
+                            elif item_type == "news":
+                                history_text = f"""News Listening History:
+Items: {len(item_data.get('news_items', []))} news items
+Topics: {', '.join([item.get('category', '') for item in item_data.get('news_items', [])[:3]])}
+Feedback: {user_feedback or 'No feedback'}
+"""
+                            elif item_type == "podcast":
+                                podcast = item_data.get("podcast", {})
+                                history_text = f"""Podcast Listening History:
+Title: {podcast.get('title', 'Unknown')}
+Host: {podcast.get('host', 'Unknown')}
+Category: {podcast.get('category', 'Unknown')}
+Feedback: {user_feedback or 'No feedback'}
+"""
+                            else:
+                                history_text = f"""Story Listening History:
+Type: {item_type}
+Feedback: {user_feedback or 'No feedback'}
+"""
+                            
+                            # Store minimal metadata to avoid chunk size issues
+                            doc = Document(
+                                text=history_text,
+                                metadata={
+                                    "type": "history",
+                                    "user_id": d.get("user_id"),
+                                    "item_type": item_type,
+                                    "timestamp": d.get("timestamp", datetime.now().isoformat()),
+                                    # Store only essential fields, not full raw_data
+                                    "feedback": user_feedback or ""
+                                }
+                            )
+                        else:
+                            # Unknown type, create basic document
+                            doc = Document(
+                                text=json.dumps(d),
+                                metadata={
+                                    "type": d.get("type", "unknown"),
+                                    "user_id": d.get("user_id"),
+                                    "timestamp": d.get("timestamp", datetime.now().isoformat()),
+                                    # Removed raw_data to reduce metadata size - essential fields stored separately
+                                }
+                            )
+                        
+                        self.documents.append(doc)
+                    except Exception as e:
+                        rag_logger.warning(f"⚠️  Skipping invalid document during load: {e}")
+                        continue
+                
+                rag_logger.info(f"📂 Loaded {len(self.documents)} documents from RAG storage")
+                print(f"📂 [RAG] Loaded {len(self.documents)} documents from storage")
+                
+                # Build vector index if embeddings are available
+                if self.documents and self.embedding_available:
+                    try:
+                        rag_logger.info(f"🔨 Building vector index from {len(self.documents)} documents...")
+                        print(f"🔨 [RAG] Building vector index from {len(self.documents)} documents...")
+                        self.index = VectorStoreIndex.from_documents(
+                            self.documents,
+                            storage_context=self.storage_context
+                        )
+                        rag_logger.info(f"✅ Vector index built successfully with {len(self.documents)} documents")
+                        print(f"✅ [RAG] Vector index built with {len(self.documents)} documents")
+                    except Exception as e:
+                        rag_logger.error(f"❌ Failed to build vector index: {e}")
+                        print(f"Warning: Could not build vector index: {e}")
+                        self.index = None
+                elif self.documents:
+                    rag_logger.info(f"ℹ️  {len(self.documents)} documents loaded but embeddings disabled - using fallback mode")
+                    print(f"ℹ️  [RAG] {len(self.documents)} documents loaded but embeddings disabled - using fallback mode")
             except Exception as e:
+                rag_logger.error(f"❌ Error loading user data: {e}")
                 print(f"Error loading user data: {e}")
     
     def _save_user_data(self):
         """Save user data to file"""
         try:
-            data = [json.loads(doc.text) for doc in self.documents]
+            data = []
+            for doc in self.documents:
+                try:
+                    # Try to get raw_data from metadata first (new format)
+                    raw_data = doc.metadata.get("raw_data")
+                    if raw_data:
+                        data.append(json.loads(raw_data))
+                    else:
+                        # Fallback: try to parse doc.text as JSON (old format)
+                        data.append(json.loads(doc.text))
+                except (json.JSONDecodeError, KeyError) as e:
+                    # Skip documents that can't be parsed
+                    rag_logger.warning(f"⚠️  Skipping document that couldn't be parsed: {e}")
+                    continue
+            
             with open(self.user_data_file, 'w') as f:
                 json.dump(data, f, indent=2)
+            rag_logger.info(f"💾 Saved {len(data)} documents to {self.user_data_file}")
         except Exception as e:
+            rag_logger.error(f"❌ Error saving user data: {e}")
+            import traceback
+            rag_logger.error(traceback.format_exc())
             print(f"Error saving user data: {e}")
     
-    def store_user_preferences(self, preferences: Dict[str, Any]):
-        """Store user preferences in RAG system"""
+    def store_user_preferences(self, preferences: Dict[str, Any], user_id: str = None):
+        """Store user preferences in RAG system with user ID"""
+        if not user_id:
+            rag_logger.warning("⚠️  Storing preferences without user_id - data will not be user-specific")
+            print("⚠️  [RAG] Warning: Storing preferences without user_id")
+        
         pref_doc = {
             "type": "preferences",
+            "user_id": user_id,
             "timestamp": datetime.now().isoformat(),
             "data": preferences
         }
         
-        doc = Document(text=json.dumps(pref_doc))
+        # Create a more descriptive document for better RAG retrieval
+        pref_text = f"""User Preferences:
+Name: {preferences.get('name', 'Unknown')}
+Favorite Genres: {', '.join(preferences.get('favorite_genres', []))}
+Interests: {', '.join(preferences.get('interests', []))}
+Podcast Interests: {', '.join(preferences.get('podcast_interests', []))}
+Mood: {preferences.get('mood', 'neutral')}
+Content Filter: Music={preferences.get('content_filter', {}).get('music', True)}, News={preferences.get('content_filter', {}).get('news', True)}, Podcasts={preferences.get('content_filter', {}).get('podcasts', True)}, Stories={preferences.get('content_filter', {}).get('stories', True)}
+"""
+        
+        doc = Document(
+            text=pref_text,
+            metadata={
+                "type": "preferences",
+                "user_id": user_id,
+                "timestamp": datetime.now().isoformat(),
+                "raw_data": json.dumps(pref_doc)
+            }
+        )
         self.documents.append(doc)
         
+        rag_logger.info(f"📝 STORING PREFERENCES: user_id={user_id}, Name={preferences.get('name')}, Genres={preferences.get('favorite_genres')}, Mood={preferences.get('mood')}")
+        print(f"📝 [RAG] Storing preferences for user {user_id} ({preferences.get('name', 'user')})")
+        
         # Rebuild index if embeddings are available
         if self.embedding_available:
             try:
@@ -98,68 +325,258 @@ class RadioRAGSystem:
                     self.documents,
                     storage_context=self.storage_context
                 )
+                rag_logger.info(f"✅ Vector index rebuilt with {len(self.documents)} documents (embeddings enabled)")
+                print(f"✅ [RAG] Index updated with {len(self.documents)} documents")
             except Exception as e:
+                rag_logger.error(f"❌ Failed to rebuild index: {e}")
                 print(f"Warning: Could not rebuild index: {e}")
                 self.index = None
+        else:
+            rag_logger.info(f"ℹ️  Preferences stored (embeddings disabled, {len(self.documents)} total documents)")
+            print(f"ℹ️  [RAG] Preferences stored (embeddings disabled)")
         
         self._save_user_data()
     
-    def store_listening_history(self, item_type: str, item_data: Dict[str, Any], user_feedback: str = None):
-        """Store listening history with optional feedback"""
+    def store_listening_history(self, item_type: str, item_data: Dict[str, Any], user_id: str = None, user_feedback: str = None):
+        """Store listening history with optional feedback and user ID"""
+        if not user_id:
+            rag_logger.warning(f"⚠️  Storing {item_type} history without user_id - data will not be user-specific")
+            print(f"⚠️  [RAG] Warning: Storing {item_type} history without user_id")
+        
         history_doc = {
             "type": "history",
+            "user_id": user_id,
             "item_type": item_type,  # music, news, podcast, story
             "timestamp": datetime.now().isoformat(),
             "data": item_data,
             "feedback": user_feedback
         }
         
-        doc = Document(text=json.dumps(history_doc))
+        # Create descriptive text for better RAG retrieval
+        if item_type == "music":
+            track = item_data.get("track", {})
+            history_text = f"""Music Listening History:
+Title: {track.get('title', 'Unknown')}
+Artist: {track.get('artist', 'Unknown')}
+Genre: {track.get('genre', 'Unknown')}
+Source: {track.get('source', 'Unknown')}
+Feedback: {user_feedback or 'No feedback'}
+"""
+            rag_logger.info(f"🎵 STORING MUSIC HISTORY: user_id={user_id}, {track.get('title', 'Unknown')} by {track.get('artist', 'Unknown')} ({track.get('genre', 'Unknown')}) - Feedback: {user_feedback or 'None'}")
+            print(f"🎵 [RAG] Storing music for user {user_id}: {track.get('title', 'Unknown')} by {track.get('artist', 'Unknown')}")
+        elif item_type == "news":
+            history_text = f"""News Listening History:
+Items: {len(item_data.get('news_items', []))} news items
+Topics: {', '.join([item.get('category', '') for item in item_data.get('news_items', [])[:3]])}
+Feedback: {user_feedback or 'No feedback'}
+"""
+            rag_logger.info(f"📰 STORING NEWS HISTORY: user_id={user_id}, {len(item_data.get('news_items', []))} items - Feedback: {user_feedback or 'None'}")
+            print(f"📰 [RAG] Storing news history for user {user_id}: {len(item_data.get('news_items', []))} items")
+        elif item_type == "podcast":
+            podcast = item_data.get("podcast", {})
+            history_text = f"""Podcast Listening History:
+Title: {podcast.get('title', 'Unknown')}
+Host: {podcast.get('host', 'Unknown')}
+Category: {podcast.get('category', 'Unknown')}
+Feedback: {user_feedback or 'No feedback'}
+"""
+            rag_logger.info(f"🎙️ STORING PODCAST HISTORY: user_id={user_id}, {podcast.get('title', 'Unknown')} - Feedback: {user_feedback or 'None'}")
+            print(f"🎙️ [RAG] Storing podcast for user {user_id}: {podcast.get('title', 'Unknown')}")
+        else:
+            history_text = f"""Story Listening History:
+Type: {item_type}
+Feedback: {user_feedback or 'No feedback'}
+"""
+            rag_logger.info(f"📖 STORING STORY HISTORY: user_id={user_id}, {item_type} - Feedback: {user_feedback or 'None'}")
+            print(f"📖 [RAG] Storing story history for user {user_id}: {item_type}")
+        
+        doc = Document(
+            text=history_text,
+            metadata={
+                "type": "history",
+                "user_id": user_id,
+                "item_type": item_type,
+                "timestamp": datetime.now().isoformat(),
+                "raw_data": json.dumps(history_doc)
+            }
+        )
         self.documents.append(doc)
         
-        # Rebuild index if embeddings are available
-        if self.embedding_available:
+        # Rebuild index if embeddings are available (but only periodically to avoid too many rebuilds)
+        # Rebuild every 5 documents or if index doesn't exist
+        if self.embedding_available and (not self.index or len(self.documents) % 5 == 0):
             try:
                 self.index = VectorStoreIndex.from_documents(
                     self.documents,
                     storage_context=self.storage_context
                 )
+                if len(self.documents) % 5 == 0:
+                    rag_logger.info(f"✅ Vector index rebuilt (total documents: {len(self.documents)})")
+                    print(f"✅ [RAG] Index updated (total documents: {len(self.documents)})")
             except Exception as e:
+                rag_logger.error(f"❌ Failed to rebuild index: {e}")
                 print(f"Warning: Could not rebuild index: {e}")
                 self.index = None
         
         self._save_user_data()
     
-    def get_user_preferences(self) -> Dict[str, Any]:
-        """Retrieve latest user preferences"""
+    def get_user_preferences(self, user_id: str = None) -> Dict[str, Any]:
+        """Retrieve latest user preferences for a specific user"""
         preferences = {}
         for doc in reversed(self.documents):
             try:
-                data = json.loads(doc.text)
-                if data.get("type") == "preferences":
-                    preferences = data.get("data", {})
-                    break
-            except:
+                # Check metadata first (new format with user_id)
+                doc_user_id = doc.metadata.get("user_id")
+                if user_id and doc_user_id != user_id:
+                    continue  # Skip documents from other users
+                
+                # Try to parse from metadata raw_data (new format)
+                raw_data = doc.metadata.get("raw_data")
+                if raw_data:
+                    data = json.loads(raw_data)
+                    if data.get("type") == "preferences":
+                        if not user_id or data.get("user_id") == user_id:
+                            preferences = data.get("data", {})
+                            break
+                else:
+                    # Fallback: try to parse doc.text as JSON (old format)
+                    data = json.loads(doc.text)
+                    if data.get("type") == "preferences":
+                        if not user_id or data.get("user_id") == user_id:
+                            preferences = data.get("data", {})
+                            break
+            except (json.JSONDecodeError, KeyError, AttributeError) as e:
+                rag_logger.debug(f"Skipping document in get_user_preferences: {e}")
                 continue
         
+        if user_id and not preferences:
+            rag_logger.warning(f"⚠️  No preferences found for user_id={user_id}")
+        
         return preferences
     
-    def get_recommendations(self, query: str) -> Dict[str, Any]:
-        """Get personalized recommendations based on user history and preferences"""
+    def get_recommendations(self, query: str, user_id: str = None) -> Dict[str, Any]:
+        """Get personalized recommendations based on user history and preferences using RAG"""
         if not self.index or not self.llm_available:
-            return self._get_default_recommendations()
+            rag_logger.warning(f"⚠️  RAG RECOMMENDATIONS UNAVAILABLE: user_id={user_id}, query='{query}' (no index or LLM)")
+            print("ℹ️  RAG recommendations unavailable (no index or LLM) - using defaults")
+            return self._get_default_recommendations(user_id=user_id)
         
         try:
-            query_engine = self.index.as_query_engine()
+            rag_logger.info(f"🔍 RAG RECOMMENDATIONS QUERY: '{query}'")
+            print(f"🔍 [RAG] Getting recommendations for: '{query[:60]}...'")
+            
+            # Use RAG to query user history and preferences
+            query_engine = self.index.as_query_engine(
+                similarity_top_k=5,  # Get top 5 relevant documents
+                response_mode="compact"  # Compact response
+            )
             response = query_engine.query(query)
             
-            return {
-                "recommendations": str(response),
-                "source": "RAG"
+            response_text = str(response)
+            rag_logger.info(f"✅ RAG RECOMMENDATIONS RESPONSE: {response_text[:200]}...")
+            print(f"✅ [RAG] LLM generated recommendations: {response_text[:150]}...")
+            
+            # Extract recommendations from RAG response
+            recommendations = {
+                "recommendations": response_text,
+                "source": "RAG",
+                "query": query
             }
+            
+            # Also try to extract structured data from response
+            response_lower = response_text.lower()
+            if "genre" in response_lower or "music" in response_lower:
+                # Try to extract genre preferences
+                for genre in ["pop", "rock", "jazz", "classical", "electronic", "hip-hop", "country", "indie", "rap", "blues", "folk"]:
+                    if genre in response_lower:
+                        recommendations.setdefault("suggested_genres", []).append(genre)
+                        rag_logger.info(f"  🎵 Extracted genre from RAG: {genre}")
+            
+            return recommendations
+        except Exception as e:
+            rag_logger.error(f"❌ RAG RECOMMENDATIONS ERROR: {e}")
+            import traceback
+            rag_logger.error(traceback.format_exc())
+            print(f"Error getting RAG recommendations: {e}")
+            traceback.print_exc()
+            return self._get_default_recommendations(user_id=user_id)
+    
+    def query_user_context(self, query: str, user_id: str = None, top_k: int = 3) -> List[Dict[str, Any]]:
+        """Query user context using vector search - returns relevant documents filtered by user_id"""
+        if not self.index or not self.embedding_available:
+            rag_logger.warning(f"⚠️  RAG QUERY SKIPPED (no index/embeddings): '{query}'")
+            print(f"⚠️  [RAG] Query skipped - embeddings not available")
+            return []
+        
+        try:
+            rag_logger.info(f"🔍 RAG QUERY: user_id={user_id}, query='{query}' (top_k={top_k})")
+            print(f"🔍 [RAG] Querying for user {user_id}: '{query[:60]}...'")
+            
+            # Retrieve more documents than needed, then filter by user_id
+            # This ensures we get top_k results for the specific user
+            retrieve_count = top_k * 3 if user_id else top_k  # Get more if filtering
+            retriever = self.index.as_retriever(similarity_top_k=retrieve_count)
+            nodes = retriever.retrieve(query)
+            
+            results = []
+            for i, node in enumerate(nodes):
+                try:
+                    # Filter by user_id if provided
+                    node_user_id = node.metadata.get("user_id")
+                    if user_id and node_user_id != user_id:
+                        continue  # Skip documents from other users
+                    
+                    score = node.score if hasattr(node, 'score') else None
+                    node_type = node.metadata.get("type", "unknown")
+                    item_type = node.metadata.get("item_type", "")
+                    
+                    data = json.loads(node.metadata.get("raw_data", "{}"))
+                    result = {
+                        "text": node.text,
+                        "score": score,
+                        "metadata": node.metadata,
+                        "data": data
+                    }
+                    results.append(result)
+                    
+                    # Log each retrieved document
+                    preview = node.text[:100].replace('\n', ' ')
+                    rag_logger.info(f"  📄 Retrieved #{len(results)}: user_id={node_user_id}, type={node_type}, item_type={item_type}, score={score:.4f if score else 'N/A'}, preview='{preview}...'")
+                    print(f"  📄 [RAG] Retrieved #{len(results)}: {node_type} (user: {node_user_id}, score: {score:.4f if score else 'N/A'}) - {preview}...")
+                    
+                    # Stop if we have enough results for this user
+                    if len(results) >= top_k:
+                        break
+                    
+                except Exception as parse_error:
+                    # Still check user_id even if parsing fails
+                    node_user_id = node.metadata.get("user_id")
+                    if user_id and node_user_id != user_id:
+                        continue
+                    
+                    score = node.score if hasattr(node, 'score') else None
+                    result = {
+                        "text": node.text,
+                        "score": score,
+                        "metadata": node.metadata
+                    }
+                    results.append(result)
+                    rag_logger.warning(f"  ⚠️  Retrieved #{len(results)} (parse error): user_id={node_user_id}, score={score}, text_preview='{node.text[:50]}...'")
+                    print(f"  ⚠️  [RAG] Retrieved #{len(results)} (parse error, user: {node_user_id}, score: {score:.4f if score else 'N/A'})")
+                    
+                    if len(results) >= top_k:
+                        break
+            
+            rag_logger.info(f"✅ RAG QUERY COMPLETE: Retrieved {len(results)} documents for user_id={user_id}")
+            print(f"✅ [RAG] Query complete: {len(results)} documents retrieved for user {user_id}")
+            
+            return results
         except Exception as e:
-            print(f"Error getting recommendations: {e}")
-            return self._get_default_recommendations()
+            rag_logger.error(f"❌ RAG QUERY ERROR: {e}")
+            import traceback
+            rag_logger.error(traceback.format_exc())
+            print(f"❌ [RAG] Query error: {e}")
+            return []
     
     def _get_default_recommendations(self) -> Dict[str, Any]:
         """Return default recommendations when RAG is not available"""