Update src/streamlit_app.py

src/streamlit_app.py

@@ -3,10 +3,8 @@ import requests
 import pandas as pd
 from together import Together
 import os
-from sklearn.feature_extraction.text import TfidfVectorizer
-from sklearn.preprocessing import StandardScaler
-from sklearn.cluster import KMeans
-import numpy as np
+import json
+from collections import defaultdict
 
 # =============================================================================
 # CONFIGURATION - Using Secrets Management
@@ -251,128 +249,54 @@ def extract_unique_names(content_list, field):
                     unique_names.add(cleaned_name)
     return sorted(list(unique_names))
 
-@st.cache_data(ttl=3600)  # Cache feature vectors for 1 hour
-def prepare_content_features(content_list):
-    """Prepare and cache content features for similarity matching"""
-    # Convert to DataFrame for easier processing
-    df = pd.DataFrame(content_list)
-    
-    # Create TF-IDF vectorizer for text features
-    tfidf = TfidfVectorizer(
-        stop_words='english',
-        max_features=1000,  # Limit features to most important ones
-        ngram_range=(1, 2)  # Include bigrams for better context
-    )
-    
-    # Combine relevant text fields with weights
-    df['text_features'] = (
-        df['listed_in'].fillna('') + ' ' +  # Genres
-        df['description'].fillna('') + ' ' +  # Plot
-        df['cast'].fillna('').apply(lambda x: ' '.join(x.split(',')[:3])) + ' ' +  # Top 3 cast members
-        df['director'].fillna('')  # Director
-    )
-    
-    # Get text features
-    text_features = tfidf.fit_transform(df['text_features'])
-    
-    # Prepare numerical features
-    df['release_year'] = pd.to_numeric(df['release_year'], errors='coerce').fillna(0)
-    
-    # Extract duration in minutes for movies
-    def extract_duration_minutes(duration):
-        if pd.isna(duration) or 'Season' in str(duration):
-            return 0
-        try:
-            return int(str(duration).split()[0])
-        except:
-            return 0
-    
-    df['duration_minutes'] = df['duration'].apply(extract_duration_minutes)
-    
-    # Scale numerical features
-    scaler = StandardScaler()
-    numerical_features = scaler.fit_transform(df[['release_year', 'duration_minutes']])
-    
-    # Convert sparse matrix to dense for easier calculations
-    text_features_dense = text_features.toarray()
-    
-    # Combine features with appropriate weights
-    combined_features = np.hstack([
-        text_features_dense * 0.8,  # 80% weight to text features
-        numerical_features * 0.2     # 20% weight to numerical features
-    ])
-    
-    return combined_features, df
-
-@st.cache_resource  # Cache the KMeans model
-def get_kmeans_model(n_clusters=50):
-    """Get or create cached KMeans model"""
-    return KMeans(
-        n_clusters=n_clusters,
-        random_state=42,
-        n_init=10,  # Number of times to run with different centroid seeds
-    )
-
-@st.cache_data(ttl=3600)  # Cache cluster assignments for 1 hour
-def get_content_clusters(features, n_clusters=50):
-    """Get cluster assignments for all content"""
-    kmeans = get_kmeans_model(n_clusters)
-    return kmeans.fit_predict(features)
-
-def find_similar_content(content, all_content_list, n_clusters=50, n_recommendations=5):
-    """Find similar content using optimized k-means clustering"""
+def get_similar_content(content, n_recommendations=5):
+    """Get pre-computed similar content from database"""
     try:
-        # Get cached features and cluster assignments
-        features, df = prepare_content_features(all_content_list)
-        cluster_labels = get_content_clusters(features, n_clusters)
-        
-        # Find the index of the selected content
-        selected_idx = df[df['title'] == content['title']].index[0]
-        selected_cluster = cluster_labels[selected_idx]
-        
-        # Get indices of content in the same cluster
-        cluster_indices = np.where(cluster_labels == selected_cluster)[0]
-        
-        # If cluster is too small, adjust n_clusters and retry
-        if len(cluster_indices) < n_recommendations + 1:
-            cluster_labels = get_content_clusters(features, n_clusters // 2)
-            selected_cluster = cluster_labels[selected_idx]
-            cluster_indices = np.where(cluster_labels == selected_cluster)[0]
-        
-        # Calculate distances to all points in the same cluster
-        cluster_features = features[cluster_indices]
-        selected_features = features[selected_idx].reshape(1, -1)
+        # Get database credentials
+        api_token, _ = get_api_credentials()
+        headers = {
+            "xc-token": api_token,
+            "accept": "application/json"
+        }
         
-        # Use euclidean distance for similarity
-        distances = np.linalg.norm(cluster_features - selected_features, axis=1)
+        # Query the similarities table
+        similarity_table_url = "https://mtoft20-potm.hf.space/api/v1/db/data/noco/p9pozkcw81t9aee/content_similarities"
+        params = {
+            "where": f"(title,eq,\"{content['title']}\")"
+        }
         
-        # Get indices of most similar content (excluding self)
-        similar_indices = cluster_indices[np.argsort(distances)][1:n_recommendations+1]
+        response = requests.get(similarity_table_url, headers=headers, params=params)
         
-        # Convert to list of content dictionaries
-        similar_content = []
-        for idx in similar_indices:
-            row = df.iloc[idx]
-            similarity_score = 1 / (1 + distances[np.where(cluster_indices == idx)[0][0]])
+        if response.status_code == 200:
+            data = response.json()
+            if data and len(data.get('list', [])) > 0:
+                # Get similar items from stored data
+                similar_items = json.loads(data['list'][0]['similar_items'])
+                
+                # Limit to requested number
+                similar_items = similar_items[:n_recommendations]
+                
+                # Get full content details for each similar item
+                similar_content = []
+                for item in similar_items:
+                    # Query main content table for full details
+                    content_params = {
+                        "where": f"(title,eq,\"{item['title']}\")"
+                    }
+                    content_response = requests.get(NOCODB_URL, headers=headers, params=content_params)
+                    
+                    if content_response.status_code == 200:
+                        content_data = content_response.json()
+                        if content_data and len(content_data.get('list', [])) > 0:
+                            content_dict = content_data['list'][0]
+                            content_dict['similarity'] = f"{item['similarity']:.2%}"
+                            similar_content.append(content_dict)
+                
+                return similar_content
             
-            content_dict = {
-                'title': row['title'],
-                'type': row['type'],
-                'release_year': row['release_year'],
-                'listed_in': row['listed_in'],
-                'description': row['description'],
-                'streaming_service': row['streaming_service'],
-                'rating': row['rating'],
-                'duration': row['duration'],
-                'cast': row['cast'],
-                'director': row['director'],
-                'similarity': f"{similarity_score:.2%}"
-            }
-            similar_content.append(content_dict)
-        
-        return similar_content
+        return []
     except Exception as e:
-        st.error(f"Error finding similar content: {str(e)}")
+        st.error(f"Error fetching similar content: {str(e)}")
         return []
 
 # =============================================================================
@@ -630,7 +554,7 @@ def main():
                     similar_button = st.button(f"🔍 Find Similar Content", key=f"similar_{i}")
                     if similar_button:
                         with st.spinner("Finding similar content..."):
-                            similar_content = find_similar_content(content, all_content)
+                            similar_content = get_similar_content(content)
                             
                             if similar_content:
                                 # Create tabs for different aspects of recommendations
@@ -656,14 +580,13 @@ def main():
                                 with sim_tab2:
                                     st.write("**Why these recommendations?**")
                                     st.write("""
-                                    These titles were selected based on our k-means clustering algorithm which groups similar content together using:
-                                    - Genre matching and themes (80% weight)
-                                    - Release year proximity (10% weight)
-                                    - Duration similarity (10% weight)
-                                    - Cast and director overlap
-                                    - Content description similarity
+                                    These recommendations are pre-computed using advanced content analysis:
+                                    - Genre and theme matching
+                                    - Plot similarity analysis
+                                    - Cast and director relationships
+                                    - Release year proximity
                                     
-                                    The percentage match indicates how close each title is to your selected content within its cluster.
+                                    The percentage match indicates how similar each title is to your selection.
                                     """)
                             else:
                                 st.info("No similar content found.")