Update src/streamlit_app.py

src/streamlit_app.py

@@ -251,17 +251,29 @@ def extract_unique_names(content_list, field):
                     unique_names.add(cleaned_name)
     return sorted(list(unique_names))
 
-def prepare_movie_features(content_list):
-    """Prepare movie features for clustering"""
+@st.cache_data(ttl=3600)  # Cache feature vectors for 1 hour
+def prepare_content_features():
+    """Prepare and cache content features for similarity matching"""
     # Convert to DataFrame for easier processing
-    df = pd.DataFrame(content_list)
+    df = pd.DataFrame(all_content)
     
-    # Create TF-IDF features from genres and description
-    tfidf = TfidfVectorizer(stop_words='english')
+    # Create TF-IDF vectorizer for text features
+    tfidf = TfidfVectorizer(
+        stop_words='english',
+        max_features=1000,  # Limit features to most important ones
+        ngram_range=(1, 2)  # Include bigrams for better context
+    )
+    
+    # Combine relevant text fields with weights
+    df['text_features'] = (
+        df['listed_in'].fillna('') + ' ' +  # Genres
+        df['description'].fillna('') + ' ' +  # Plot
+        df['cast'].fillna('').apply(lambda x: ' '.join(x.split(',')[:3])) + ' ' +  # Top 3 cast members
+        df['director'].fillna('')  # Director
+    )
     
-    # Combine genres and description for text features
-    df['text_features'] = df['listed_in'].fillna('') + ' ' + df['description'].fillna('')
-    text_features = tfidf.fit_transform(df['text_features']).toarray()
+    # Get text features
+    text_features = tfidf.fit_transform(df['text_features'])
     
     # Prepare numerical features
     df['release_year'] = pd.to_numeric(df['release_year'], errors='coerce').fillna(0)
@@ -281,37 +293,87 @@ def prepare_movie_features(content_list):
     scaler = StandardScaler()
     numerical_features = scaler.fit_transform(df[['release_year', 'duration_minutes']])
     
-    # Combine all features
-    all_features = np.hstack([text_features, numerical_features])
-    
-    return all_features, df
-
-def find_similar_content(content_list, selected_content, n_clusters=20, n_recommendations=5):
-    """Find similar content using k-means clustering"""
-    # Prepare features
-    features, df = prepare_movie_features(content_list)
-    
-    # Perform k-means clustering
-    kmeans = KMeans(n_clusters=n_clusters, random_state=42)
-    cluster_labels = kmeans.fit_predict(features)
-    
-    # Find the cluster of the selected content
-    selected_idx = df[df['title'] == selected_content['title']].index[0]
-    selected_cluster = cluster_labels[selected_idx]
+    # Convert sparse matrix to dense for easier calculations
+    text_features_dense = text_features.toarray()
     
-    # Get indices of content in the same cluster
-    cluster_indices = np.where(cluster_labels == selected_cluster)[0]
+    # Combine features with appropriate weights
+    combined_features = np.hstack([
+        text_features_dense * 0.8,  # 80% weight to text features
+        numerical_features * 0.2     # 20% weight to numerical features
+    ])
     
-    # Calculate distances to all points in the same cluster
-    cluster_features = features[cluster_indices]
-    selected_features = features[selected_idx].reshape(1, -1)
-    distances = np.linalg.norm(cluster_features - selected_features, axis=1)
-    
-    # Get the indices of the most similar content (excluding the selected content)
-    similar_indices = cluster_indices[np.argsort(distances)][1:n_recommendations+1]
-    
-    # Return the similar content
-    return [content_list[idx] for idx in similar_indices]
+    return combined_features, df
+
+@st.cache_resource  # Cache the KMeans model
+def get_kmeans_model(n_clusters=50):
+    """Get or create cached KMeans model"""
+    return KMeans(
+        n_clusters=n_clusters,
+        random_state=42,
+        n_init=10,  # Number of times to run with different centroid seeds
+    )
+
+@st.cache_data(ttl=3600)  # Cache cluster assignments for 1 hour
+def get_content_clusters(features, n_clusters=50):
+    """Get cluster assignments for all content"""
+    kmeans = get_kmeans_model(n_clusters)
+    return kmeans.fit_predict(features)
+
+def find_similar_content(content, n_clusters=50, n_recommendations=5):
+    """Find similar content using optimized k-means clustering"""
+    try:
+        # Get cached features and cluster assignments
+        features, df = prepare_content_features()
+        cluster_labels = get_content_clusters(features, n_clusters)
+        
+        # Find the index of the selected content
+        selected_idx = df[df['title'] == content['title']].index[0]
+        selected_cluster = cluster_labels[selected_idx]
+        
+        # Get indices of content in the same cluster
+        cluster_indices = np.where(cluster_labels == selected_cluster)[0]
+        
+        # If cluster is too small, adjust n_clusters and retry
+        if len(cluster_indices) < n_recommendations + 1:
+            cluster_labels = get_content_clusters(features, n_clusters // 2)
+            selected_cluster = cluster_labels[selected_idx]
+            cluster_indices = np.where(cluster_labels == selected_cluster)[0]
+        
+        # Calculate distances to all points in the same cluster
+        cluster_features = features[cluster_indices]
+        selected_features = features[selected_idx].reshape(1, -1)
+        
+        # Use euclidean distance for similarity
+        distances = np.linalg.norm(cluster_features - selected_features, axis=1)
+        
+        # Get indices of most similar content (excluding self)
+        similar_indices = cluster_indices[np.argsort(distances)][1:n_recommendations+1]
+        
+        # Convert to list of content dictionaries
+        similar_content = []
+        for idx in similar_indices:
+            row = df.iloc[idx]
+            similarity_score = 1 / (1 + distances[np.where(cluster_indices == idx)[0][0]])
+            
+            content_dict = {
+                'title': row['title'],
+                'type': row['type'],
+                'release_year': row['release_year'],
+                'listed_in': row['listed_in'],
+                'description': row['description'],
+                'streaming_service': row['streaming_service'],
+                'rating': row['rating'],
+                'duration': row['duration'],
+                'cast': row['cast'],
+                'director': row['director'],
+                'similarity': f"{similarity_score:.2%}"
+            }
+            similar_content.append(content_dict)
+        
+        return similar_content
+    except Exception as e:
+        st.error(f"Error finding similar content: {str(e)}")
+        return []
 
 # =============================================================================
 # MAIN APP
@@ -564,18 +626,45 @@ def main():
                     st.write(f"**📖 Description:**")
                     st.write(content.get('description', 'N/A'))
                     
-                    # Add Find Similar button
-                    if st.button(f"🔍 Find Similar Content", key=f"similar_{i}"):
+                    # Add Find Similar button with loading state
+                    similar_button = st.button(f"🔍 Find Similar Content", key=f"similar_{i}")
+                    if similar_button:
                         with st.spinner("Finding similar content..."):
-                            similar_content = find_similar_content(all_content, content)
+                            similar_content = find_similar_content(content)
                             
                             if similar_content:
-                                st.write("**👥 Similar Content You Might Like:**")
-                                for sim_content in similar_content:
-                                    with st.container():
-                                        st.write(f"**{sim_content.get('title')}** ({sim_content.get('type')}, {sim_content.get('release_year')})")
-                                        st.write(f"*Genres:* {sim_content.get('listed_in')}")
-                                        st.write("---")
+                                # Create tabs for different aspects of recommendations
+                                sim_tab1, sim_tab2 = st.tabs(["📺 Similar Titles", "🔍 Why These Recommendations"])
+                                
+                                with sim_tab1:
+                                    for sim_content in similar_content:
+                                        with st.container():
+                                            col1, col2 = st.columns([3, 1])
+                                            with col1:
+                                                st.write(f"**{sim_content.get('title')}** ({sim_content.get('type')}, {sim_content.get('release_year')})")
+                                                st.write(f"*Available on:* {sim_content.get('streaming_service')}")
+                                                st.write(f"*Genres:* {sim_content.get('listed_in')}")
+                                            with col2:
+                                                st.write(f"**Match:** {sim_content.get('similarity', 'N/A')}")
+                                            
+                                            with st.expander("See more details"):
+                                                st.write(f"**Cast:** {sim_content.get('cast', 'N/A')}")
+                                                st.write(f"**Director:** {sim_content.get('director', 'N/A')}")
+                                                st.write(f"**Description:** {sim_content.get('description', 'N/A')}")
+                                            st.write("---")
+                                
+                                with sim_tab2:
+                                    st.write("**Why these recommendations?**")
+                                    st.write("""
+                                    These titles were selected based on our k-means clustering algorithm which groups similar content together using:
+                                    - Genre matching and themes (80% weight)
+                                    - Release year proximity (10% weight)
+                                    - Duration similarity (10% weight)
+                                    - Cast and director overlap
+                                    - Content description similarity
+                                    
+                                    The percentage match indicates how close each title is to your selected content within its cluster.
+                                    """)
                             else:
                                 st.info("No similar content found.")