Update app.py

app.py

@@ -1,137 +1,367 @@
-import pandas as pd
 import numpy as np
-from surprise import SVD, Dataset, Reader, accuracy
-from surprise.model_selection import train_test_split, cross_validate
-from collections import defaultdict
-
-class MovieRecommender:
-    def __init__(self, ratings_path, movies_path):
-        # Load data
-        self.ratings = pd.read_csv(ratings_path)
-        self.movies = pd.read_csv(movies_path)
-        
-        # Build Surprise dataset
-        reader = Reader(rating_scale=(0.5, 5.0))
-        self.data = Dataset.load_from_df(
-            self.ratings[['userId', 'movieId', 'rating']], 
-            reader
-        )
+import pandas as pd
+from scipy.sparse.linalg import svds
+from scipy.sparse import csr_matrix
+from sklearn.metrics.pairwise import cosine_similarity
+from sklearn.model_selection import train_test_split
+import warnings
+warnings.filterwarnings('ignore')
+
+# ============================================================================
+# DATA LOADING & PREPROCESSING
+# ============================================================================
+
+def load_movielens_data(ratings_path='ratings.csv', movies_path='movies.csv'):
+    """Load and prepare MovieLens data"""
+    ratings = pd.read_csv(ratings_path)
+    movies = pd.read_csv(movies_path)
+    return ratings, movies
+
+def create_user_item_matrix(ratings):
+    """Create user-item rating matrix"""
+    user_item_matrix = ratings.pivot_table(
+        index='userId', 
+        columns='movieId', 
+        values='rating'
+    ).fillna(0)
+    return user_item_matrix
+
+# ============================================================================
+# COLLABORATIVE FILTERING - USER BASED
+# ============================================================================
+
+class UserBasedCF:
+    def __init__(self, user_item_matrix):
+        self.matrix = user_item_matrix
+        self.user_similarity = None
         
-        # Train model
-        self.trainset = self.data.build_full_trainset()
-        self.algo = SVD(n_factors=100, n_epochs=20, lr_all=0.005, reg_all=0.02)
-        self.algo.fit(self.trainset)
-        
-    def recommend_movies(self, user_id, N):
-        # Get all movie IDs
-        all_movie_ids = self.movies['movieId'].unique()
-        
-        # Get movies user has already rated
-        rated_movies = self.ratings[self.ratings['userId'] == user_id]['movieId'].values
-        
-        # Get unrated movies
-        unrated_movies = [m for m in all_movie_ids if m not in rated_movies]
-        
-        # Predict ratings
-        predictions = []
-        for movie_id in unrated_movies:
-            pred = self.algo.predict(user_id, movie_id)
-            predictions.append((movie_id, pred.est))
-        
-        # Sort by predicted rating
-        predictions.sort(key=lambda x: x[1], reverse=True)
-        
-        # Get top N
-        top_n = predictions[:N]
-        
-        # Merge with movie titles
-        results = []
-        for movie_id, score in top_n:
-            title = self.movies[self.movies['movieId'] == movie_id]['title'].values[0]
-            results.append({
-                'movieId': movie_id,
-                'title': title,
-                'predicted_rating': round(score, 2)
-            })
-        
-        return results
-    
-    def evaluate(self):
-        # Cross-validation
-        results = cross_validate(
-            self.algo, 
-            self.data, 
-            measures=['RMSE', 'MAE'],
-            cv=5,
-            verbose=False
-        )
+    def fit(self):
+        """Compute user similarity matrix"""
+        self.user_similarity = cosine_similarity(self.matrix)
+        np.fill_diagonal(self.user_similarity, 0)
         
-        # Custom metrics: Precision@K, Recall@K, NDCG@K
-        trainset, testset = train_test_split(self.data, test_size=0.2)
-        self.algo.fit(trainset)
-        predictions = self.algo.test(testset)
+    def predict(self, user_id, k=50):
+        """Predict ratings for user"""
+        if user_id not in self.matrix.index:
+            return pd.Series()
         
-        # Calculate Precision@K and Recall@K
-        k = 10
-        threshold = 4.0
+        user_idx = self.matrix.index.get_loc(user_id)
+        similar_users_idx = np.argsort(self.user_similarity[user_idx])[::-1][:k]
         
-        user_est_true = defaultdict(list)
-        for uid, _, true_r, est, _ in predictions:
-            user_est_true[uid].append((est, true_r))
+        similar_users_ratings = self.matrix.iloc[similar_users_idx]
+        weights = self.user_similarity[user_idx][similar_users_idx]
         
-        precisions = []
-        recalls = []
+        weighted_sum = (similar_users_ratings.T * weights).sum(axis=1)
+        weight_sum = np.abs(weights).sum()
         
-        for uid, user_ratings in user_est_true.items():
-            user_ratings.sort(key=lambda x: x[0], reverse=True)
-            top_k = user_ratings[:k]
+        predictions = weighted_sum / (weight_sum + 1e-10)
+        user_ratings = self.matrix.loc[user_id]
+        predictions[user_ratings > 0] = 0
+        
+        return predictions
+
+# ============================================================================
+# COLLABORATIVE FILTERING - ITEM BASED
+# ============================================================================
+
+class ItemBasedCF:
+    def __init__(self, user_item_matrix):
+        self.matrix = user_item_matrix
+        self.item_similarity = None
+        
+    def fit(self):
+        """Compute item similarity matrix"""
+        self.item_similarity = cosine_similarity(self.matrix.T)
+        np.fill_diagonal(self.item_similarity, 0)
+        
+    def predict(self, user_id, k=50):
+        """Predict ratings for user"""
+        if user_id not in self.matrix.index:
+            return pd.Series()
+        
+        user_ratings = self.matrix.loc[user_id]
+        rated_items = user_ratings[user_ratings > 0]
+        
+        predictions = pd.Series(0, index=self.matrix.columns)
+        
+        for item_id in rated_items.index:
+            item_idx = self.matrix.columns.get_loc(item_id)
+            similar_items_idx = np.argsort(self.item_similarity[item_idx])[::-1][:k]
+            
+            for sim_idx in similar_items_idx:
+                sim_item_id = self.matrix.columns[sim_idx]
+                if user_ratings[sim_item_id] == 0:
+                    predictions[sim_item_id] += (
+                        self.item_similarity[item_idx][sim_idx] * rated_items[item_id]
+                    )
+        
+        predictions[user_ratings > 0] = 0
+        return predictions
+
+# ============================================================================
+# MATRIX FACTORIZATION - SVD
+# ============================================================================
+
+class SVDRecommender:
+    def __init__(self, user_item_matrix, n_factors=50):
+        self.matrix = user_item_matrix
+        self.n_factors = n_factors
+        self.user_factors = None
+        self.item_factors = None
+        self.mean_rating = None
+        
+    def fit(self):
+        """Perform SVD decomposition"""
+        matrix_centered = self.matrix.values - self.matrix.values.mean()
+        U, sigma, Vt = svds(matrix_centered, k=self.n_factors)
+        
+        self.user_factors = U
+        self.item_factors = Vt.T
+        self.sigma = np.diag(sigma)
+        self.mean_rating = self.matrix.values.mean()
+        
+        predicted = np.dot(np.dot(U, self.sigma), Vt) + self.mean_rating
+        self.predictions = pd.DataFrame(
+            predicted,
+            index=self.matrix.index,
+            columns=self.matrix.columns
+        )
+        
+    def predict(self, user_id):
+        """Get predictions for user"""
+        if user_id not in self.predictions.index:
+            return pd.Series()
+        
+        user_predictions = self.predictions.loc[user_id]
+        user_ratings = self.matrix.loc[user_id]
+        user_predictions[user_ratings > 0] = 0
+        
+        return user_predictions
+
+# ============================================================================
+# EVALUATION METRICS
+# ============================================================================
+
+def precision_at_k(recommended, relevant, k):
+    """Calculate Precision@K"""
+    recommended_k = set(recommended[:k])
+    relevant_set = set(relevant)
+    return len(recommended_k & relevant_set) / k if k > 0 else 0
+
+def recall_at_k(recommended, relevant, k):
+    """Calculate Recall@K"""
+    recommended_k = set(recommended[:k])
+    relevant_set = set(relevant)
+    return len(recommended_k & relevant_set) / len(relevant_set) if len(relevant_set) > 0 else 0
+
+def ndcg_at_k(recommended, relevant, k):
+    """Calculate NDCG@K"""
+    dcg = 0
+    for i, item in enumerate(recommended[:k]):
+        if item in relevant:
+            dcg += 1 / np.log2(i + 2)
+    
+    idcg = sum([1 / np.log2(i + 2) for i in range(min(len(relevant), k))])
+    return dcg / idcg if idcg > 0 else 0
+
+def evaluate_model(model, test_data, user_item_matrix, k=10, threshold=4.0):
+    """Evaluate model on test set"""
+    precisions, recalls, ndcgs = [], [], []
+    
+    test_users = test_data['userId'].unique()[:100]  # Sample for speed
+    
+    for user_id in test_users:
+        if user_id not in user_item_matrix.index:
+            continue
             
-            n_rel = sum(1 for (_, true_r) in user_ratings if true_r >= threshold)
-            n_rec_k = sum(1 for (est, _) in top_k if est >= threshold)
-            n_rel_and_rec_k = sum(1 for (est, true_r) in top_k 
-                                  if true_r >= threshold and est >= threshold)
+        user_test = test_data[test_data['userId'] == user_id]
+        relevant_items = user_test[user_test['rating'] >= threshold]['movieId'].tolist()
+        
+        if len(relevant_items) == 0:
+            continue
+        
+        predictions = model.predict(user_id)
+        if len(predictions) == 0:
+            continue
             
-            precisions.append(n_rel_and_rec_k / n_rec_k if n_rec_k > 0 else 0)
-            recalls.append(n_rel_and_rec_k / n_rel if n_rel > 0 else 0)
+        recommended = predictions.sort_values(ascending=False).index[:k].tolist()
         
-        return {
-            'rmse': np.mean(results['test_rmse']),
-            'mae': np.mean(results['test_mae']),
-            f'precision@{k}': np.mean(precisions),
-            f'recall@{k}': np.mean(recalls)
-        }
+        precisions.append(precision_at_k(recommended, relevant_items, k))
+        recalls.append(recall_at_k(recommended, relevant_items, k))
+        ndcgs.append(ndcg_at_k(recommended, relevant_items, k))
+    
+    return {
+        'Precision@K': np.mean(precisions),
+        'Recall@K': np.mean(recalls),
+        'NDCG@K': np.mean(ndcgs)
+    }
+
+# ============================================================================
+# RECOMMENDATION FUNCTION
+# ============================================================================
+
+def recommend_movies(user_id, N, model, movies_df):
+    """
+    Recommend top N movies for user
+    
+    Parameters:
+    - user_id: target user ID
+    - N: number of recommendations
+    - model: trained recommendation model
+    - movies_df: movies dataframe with titles
+    
+    Returns:
+    - DataFrame with movie recommendations
+    """
+    predictions = model.predict(user_id)
+    
+    if len(predictions) == 0:
+        return pd.DataFrame(columns=['movieId', 'title', 'predicted_rating'])
+    
+    top_n = predictions.sort_values(ascending=False).head(N)
+    recommendations = pd.DataFrame({
+        'movieId': top_n.index,
+        'predicted_rating': top_n.values
+    })
+    
+    recommendations = recommendations.merge(movies_df[['movieId', 'title']], on='movieId')
+    return recommendations[['movieId', 'title', 'predicted_rating']]
 
+# ============================================================================
+# MAIN EXECUTION PIPELINE
+# ============================================================================
+
+def main():
+    print("Loading data...")
+    ratings, movies = load_movielens_data()
+    
+    # Train-test split
+    train_data, test_data = train_test_split(ratings, test_size=0.2, random_state=42)
+    
+    print("Creating user-item matrix...")
+    user_item_matrix = create_user_item_matrix(train_data)
+    
+    # Train models
+    print("\n1. Training User-Based CF...")
+    user_cf = UserBasedCF(user_item_matrix)
+    user_cf.fit()
+    metrics_user_cf = evaluate_model(user_cf, test_data, user_item_matrix)
+    print(f"User-Based CF Metrics: {metrics_user_cf}")
+    
+    print("\n2. Training Item-Based CF...")
+    item_cf = ItemBasedCF(user_item_matrix)
+    item_cf.fit()
+    metrics_item_cf = evaluate_model(item_cf, test_data, user_item_matrix)
+    print(f"Item-Based CF Metrics: {metrics_item_cf}")
+    
+    print("\n3. Training SVD...")
+    svd = SVDRecommender(user_item_matrix, n_factors=50)
+    svd.fit()
+    metrics_svd = evaluate_model(svd, test_data, user_item_matrix)
+    print(f"SVD Metrics: {metrics_svd}")
+    
+    # Compare models
+    print("\n" + "="*60)
+    print("MODEL COMPARISON")
+    print("="*60)
+    comparison = pd.DataFrame({
+        'User-Based CF': metrics_user_cf,
+        'Item-Based CF': metrics_item_cf,
+        'SVD': metrics_svd
+    })
+    print(comparison)
+    
+    # Select best model (based on NDCG)
+    best_model_name = comparison.loc['NDCG@K'].idxmax()
+    print(f"\nBest Model: {best_model_name}")
+    
+    if best_model_name == 'User-Based CF':
+        best_model = user_cf
+    elif best_model_name == 'Item-Based CF':
+        best_model = item_cf
+    else:
+        best_model = svd
+    
+    # Example recommendation
+    print("\n" + "="*60)
+    print("EXAMPLE RECOMMENDATIONS")
+    print("="*60)
+    sample_user = user_item_matrix.index[0]
+    recommendations = recommend_movies(sample_user, 10, best_model, movies)
+    print(f"\nTop 10 recommendations for User {sample_user}:")
+    print(recommendations.to_string(index=False))
+    
+    return best_model, user_item_matrix, movies
+
+if __name__ == "__main__":
+    best_model, user_item_matrix, movies = main()
+
+# save_model.py
+import pickle
+import os
+
+def save_recommendation_system(model, user_item_matrix, movies, output_dir='recommendation_model'):
+    """Save trained model and data"""
+    os.makedirs(output_dir, exist_ok=True)
+    
+    with open(f'{output_dir}/model.pkl', 'wb') as f:
+        pickle.dump(model, f)
+    
+    with open(f'{output_dir}/user_item_matrix.pkl', 'wb') as f:
+        pickle.dump(user_item_matrix, f)
+    
+    movies.to_csv(f'{output_dir}/movies.csv', index=False)
+    
+    print(f"Model saved to {output_dir}/")
+
+# Save after training
+save_recommendation_system(best_model, user_item_matrix, movies)
 
 import gradio as gr
+import pickle
+import pandas as pd
 
-# Initialize recommender
-recommender = MovieRecommender('ratings.csv', 'movies.csv')
+# Load model
+with open('model.pkl', 'rb') as f:
+    model = pickle.load(f)
 
-def recommend_interface(user_id, n_recommendations):
+with open('user_item_matrix.pkl', 'rb') as f:
+    user_item_matrix = pickle.load(f)
+
+movies = pd.read_csv('movies.csv')
+
+def recommend_movies(user_id, N):
+    """Recommendation function for Gradio"""
     try:
         user_id = int(user_id)
-        n_recommendations = int(n_recommendations)
+        N = int(N)
         
-        recommendations = recommender.recommend_movies(user_id, n_recommendations)
+        if user_id not in user_item_matrix.index:
+            return "User ID not found"
         
-        output = []
-        for i, rec in enumerate(recommendations, 1):
-            output.append(f"{i}. {rec['title']} (Predicted: {rec['predicted_rating']})")
+        predictions = model.predict(user_id)
+        top_n = predictions.sort_values(ascending=False).head(N)
         
-        return "\n".join(output)
+        recommendations = pd.DataFrame({
+            'movieId': top_n.index,
+            'predicted_rating': top_n.values
+        })
+        
+        recommendations = recommendations.merge(movies[['movieId', 'title']], on='movieId')
+        return recommendations[['title', 'predicted_rating']]
+    
     except Exception as e:
         return f"Error: {str(e)}"
 
-# Create interface
-demo = gr.Interface(
-    fn=recommend_interface,
+interface = gr.Interface(
+    fn=recommend_movies,
     inputs=[
-        gr.Textbox(label="User ID", placeholder="Enter user ID"),
-        gr.Slider(minimum=1, maximum=20, value=10, step=1, label="Number of Recommendations")
+        gr.Number(label="User ID"),
+        gr.Number(label="Number of Recommendations", value=10)
     ],
-    outputs=gr.Textbox(label="Recommendations", lines=15),
+    outputs=gr.Dataframe(label="Recommended Movies"),
     title="MovieLens Recommendation System",
-    description="Enter a user ID to get personalized movie recommendations"
+    description="Enter User ID and number of recommendations"
 )
 
-demo.launch()
+interface.launch()