Chatbot working and well! Updated system prompt, sources and explanations in chat

app.py

@@ -40,9 +40,18 @@ def main():
             st.stop()
         st.session_state.generator = generator
         st.session_state.chat_history = []
+        st.session_state.current_lyrics = None
+    
+    # Display chat history
+    for message in st.session_state.chat_history:
+        user_msg, assistant_msg = message
+        with st.chat_message("user"):
+            st.write(user_msg)
+        with st.chat_message("assistant"):
+            st.markdown(f"```\n{assistant_msg}\n```")
     
     # Chat interface
-    user_input = st.chat_input("Enter your prompt...")
+    user_input = st.chat_input("Enter your prompt (ask for new lyrics or modify existing ones)...")
     
     if user_input:
         with st.chat_message("user"):
@@ -55,11 +64,38 @@ def main():
                         user_input,
                         st.session_state.chat_history
                     )
-                st.markdown(f"```\n{response['answer']}\n```")
                 
-                # Display sources
-                with st.expander("View Sources"):
+                # Store the response
+                lyrics = response['answer']
+                st.markdown(f"```\n{lyrics}\n```")
+                st.session_state.current_lyrics = lyrics
+                
+                # Display sources with content
+                with st.expander("View Sources and Context"):
+                    st.write("### Top Retrieved Contexts")
+                    for detail in response["context_details"]:
+                        st.write(f"\n**{detail['artist']} - {detail['song']}** "
+                                f"(Similarity: {detail['similarity']}%)")
+                        st.write("Content snippet:")
+                        st.text(detail['content'])
+                        st.write("---")
+                    
+                    st.write("\n### All Similar Sources")
+                    # Deduplicate sources by artist and song title
+                    seen_sources = set()
+                    unique_sources = []
+                    
                     for doc, score in response["source_documents_with_scores"]:
+                        source_key = (
+                            doc.metadata['artist'],
+                            doc.metadata['song_title']
+                        )
+                        if source_key not in seen_sources:
+                            seen_sources.add(source_key)
+                            unique_sources.append((doc, score))
+                    
+                    # Display unique sources
+                    for doc, score in unique_sources:
                         similarity = round((1 - score) * 100, 2)
                         st.write(
                             f"- {doc.metadata['artist']} - "
@@ -67,8 +103,9 @@ def main():
                             f"(Similarity: {similarity}%)"
                         )
                 
+                # Update chat history
                 st.session_state.chat_history.append(
-                    (user_input, response["answer"])
+                    (user_input, lyrics)
                 )
                 
             except Exception as e:

src/generator/generator.py

@@ -1,7 +1,7 @@
 from typing import Dict, List, Optional
 from pathlib import Path
 from langchain_openai import OpenAIEmbeddings, ChatOpenAI
-from langchain_community.vectorstores import Chroma
+from langchain_chroma import Chroma
 from langchain.chains import ConversationalRetrievalChain
 from langchain.prompts import PromptTemplate
 import json
@@ -48,9 +48,9 @@ class LyricGenerator:
             
     def _setup_qa_chain(self) -> None:
         """Initialize the QA chain for generating lyrics"""
-        # Configure retriever
+        # Configure retriever with a more reasonable number of documents
         retriever = self.vector_store.as_retriever(
-            search_kwargs={"k": 100}
+            search_kwargs={"k": 20}  # Reduced from 100 to 20
         )
         
         # Create document prompt
@@ -60,51 +60,76 @@ class LyricGenerator:
         )
         
         # System prompt template
-        system_template = """You are an expert pop and hip-hop lyricist who excels at modern hits and lyric writing:
-        
-        Unless specifically requested otherwise, your lyrics should follow these guidelines:
-        
-        You are most importantly cool, and young, and up-to-date with current cultural and musical trends. 
-        
-        You will never use any cheesy, old-school, or overdone lyrical tropes. 
+        system_template = """You are an expert lyricist who excels at analyzing and 
+        adapting patterns from existing lyrics while maintaining professional 
+        songwriting standards.
 
-        For Pop Structure:
-        1. Mathematical precision in section lengths (8, 16, 32 bars)
-        2. Strategic placement of hooks and payoff lines
-        3. Clear sectional contrasts (verse/pre-chorus/chorus dynamics)
-        4. Melodic phrasing considerations
-        5. Emphasis on memorable, repeatable phrases
-        6. Building tension and release patterns
-        
-        For Hip-Hop:
-        1. Complex rhyme schemes (multisyllabic, internal) for verses
-        2. Wordplay and metaphors
-        3. Flow patterns matching modern cadences
-        4. Authentic voice and contemporary language
-        5. For Chorus:
-            - Simple, memorable, and hooky
-            - Follows Pop structure guidelines
-        6. You will not sound like anyone else.
-        7. You will not sound like you are a 90's rapper.
-        
-        Pop Song Architecture:
-        - Compact, focused verses (8-16 bars)
-        - Pre-chorus build-up (4-8 bars)
-        - Strong chorus payoff (8 bars)
-        - Bridge contrasts (8 bars)
-        - Strategic repetition
-        
-        When writing/modifying:
-        - Balance complexity with accessibility
-        - Consider melodic potential
-        - Focus on emotional peaks
-        - Mark sections clearly [Verse/Chorus/etc]
-        
-        Context: {context}
-        Chat History: {chat_history}
+        CONTEXT UTILIZATION:
+        1. Analysis of Retrieved Examples:
+           - Study rhyme patterns, flow structures, and wordplay
+           - Identify unique stylistic elements matching the requested genre
+           - Extract metaphors and imagery that fit the context
+           - Note how similar themes are handled in different styles
+        
+        2. Technical Song Construction:
+           - Mathematical precision in section lengths (8, 16, 32 bars)
+           - Strategic placement of hooks and payoff lines
+           - Clear sectional contrasts (verse/pre-chorus/chorus dynamics)
+           - Melodic phrasing considerations
+           - Build tension and release patterns
+        
+        3. Genre-Specific Excellence:
+           Pop Structure:
+           - Compact, focused verses (8-16 bars)
+           - Pre-chorus build-up (4-8 bars)
+           - Strong chorus payoff (8 bars)
+           - Bridge contrasts (8 bars)
+           - Strategic repetition
+           - Emphasis on memorable, repeatable phrases
+           
+           Hip-Hop Elements:
+           - Complex rhyme schemes (multisyllabic, internal)
+           - Advanced wordplay and metaphors
+           - Flow patterns matching contemporary cadences
+           - Authentic voice and modern language
+           - Hooks that balance complexity with accessibility
+        
+        GENERATION APPROACH:
+        1. First, analyze retrieved lyrics for:
+           - Successful patterns and techniques
+           - Thematic handling
+           - Style-specific elements
+           - Unique devices worth adapting
+        
+        2. Then craft content that:
+           - Adapts identified patterns thoughtfully
+           - Maintains professional structure
+           - Blends inspiration with innovation
+           - Stays current and authentic
+           - Avoids dated or overdone tropes
+        
+        3. Always ensure:
+           - Technical excellence in rhythm and flow
+           - Proper section marking [Verse/Chorus/etc]
+           - Emotional resonance and authenticity
+           - Modern, fresh perspective
+           - Strategic use of space and silence
+        
+        Previous Chat History:
+        {chat_history}
+        
+        Retrieved Examples for Analysis:
+        {context}
+        
+        Response Format:
+        1. Brief analysis of relevant patterns from retrieved examples
+        2. Generated lyrics with clear section marking
+        3. Explanation of how you adapted specific elements from the sources
+        
         User Request: {question}
         
-        Response: Let me generate lyrics based on these guidelines and the provided context."""
+        Response: Let me analyze the retrieved lyrics and craft something that 
+        combines their strongest elements with professional songwriting principles."""
         
         prompt = PromptTemplate(
             input_variables=["context", "chat_history", "question"],
@@ -152,23 +177,35 @@ class LyricGenerator:
             chat_history = []
             
         try:
-            # Get documents with scores
+            # Get source documents with scores first
             docs_and_scores = self.vector_store.similarity_search_with_score(
                 prompt,
-                k=50
+                k=20
             )
             
             # Sort by similarity (convert distance to similarity)
             docs_and_scores.sort(key=lambda x: x[1], reverse=False)
             
-            # Generate response
-            response = self.qa_chain({
+            # Create detailed context log
+            context_details = []
+            for doc, score in docs_and_scores[:5]:  # Log top 5 for brevity
+                similarity = round((1 - score) * 100, 2)
+                context_details.append({
+                    'artist': doc.metadata['artist'],
+                    'song': doc.metadata['song_title'],
+                    'similarity': similarity,
+                    'content': doc.page_content[:200] + "..."  # First 200 chars
+                })
+            
+            # Generate response using invoke
+            response = self.qa_chain.invoke({
                 "question": prompt,
                 "chat_history": chat_history
             })
             
-            # Add source documents with scores
+            # Add detailed context to response
             response["source_documents_with_scores"] = docs_and_scores
+            response["context_details"] = context_details
             
             return response