Progress

app.py

@@ -3,6 +3,9 @@ from course_project.application_tools import get_recommendations_text, get_data
 
 # Load app with: streamlit run course_project/app.py
 
+COSINE_SIMILARITY = 0
+EUCLIDEAN_SIMILARITY = 1
+
 # Cargar los datos de películas para mostrar en la interfaz
 movies_data = get_data()
 movies_titles = movies_data['title'].tolist()
@@ -34,6 +37,6 @@ n_results = st.number_input('Number of recommendations', min_value=1, max_value=
 
 if st.button('Get Recommendations'):
     # st.write("Debugging Suggestions:", suggestions)
-    recommendations = get_recommendations_text(suggestions, n_results=n_results)
+    recommendations = get_recommendations_text(suggestions, n_results=n_results, similarity=EUCLIDEAN_SIMILARITY)
     # st.write("Debugging Recommendations:\n", recommendations)
     st.markdown(recommendations.replace("\n", "\n\n"))

course_project/application_tools.py

@@ -6,8 +6,11 @@ import pandas as pd
 DATA_FOLDER_LOCATION = "course_project/data/"
 OLLAMA_EMBEDDING_BK_LOCATION = DATA_FOLDER_LOCATION + "ollama_embedding_bk_location.pkl"
 
+COSINE_SIMILARITY = 0
+EUCLIDEAN_SIMILARITY = 1
+
 def get_data(overwrite_db=False):
-    movies_data = data_persistence.get_db()
+    movies_data = data_persistence.get_transformed_db()
     if not isinstance(movies_data, pd.DataFrame) or overwrite_db:
         movies_data = data_import.read_or_generate_movies_data()
         movies_data["vector"] = \
@@ -20,7 +23,7 @@ def get_data(overwrite_db=False):
         movies_data["vector"] = embedding.encode_text_series_ollama_method(movies_data["vector"], file_backup_location=OLLAMA_EMBEDDING_BK_LOCATION, batch_size=500)
         print(movies_data["vector"])
 
-        data_persistence.save_db(movies_data)
+        data_persistence.save_transformed_db(movies_data)
     return movies_data
 
 def replace_ids_with_titles(suggestions, movies_data):
@@ -29,9 +32,9 @@ def replace_ids_with_titles(suggestions, movies_data):
         suggestions[user] = movie_title
     return suggestions
 
-def get_recommendations_df(suggestions, n_results):
+def get_recommendations_df(suggestions, n_results, similarity=COSINE_SIMILARITY):
     movies_data = get_data()
-    recommendations_raw = embedding.find_recommendations(suggestions, movies_data["vector"], n_results=n_results)
+    recommendations_raw = embedding.find_recommendations(suggestions, movies_data["vector"], n_results=n_results, similarity=similarity)
     recommendations_indexes = [recommendation["movie_index"] for recommendation in recommendations_raw]
     columns_of_interest = ["title", "director", "genres", "cast", "overview", "rating", "poster_path", "imdb_id"]
     recommendations = (movies_data.loc[recommendations_indexes])[columns_of_interest].copy()
@@ -50,13 +53,15 @@ def recommendations_to_text(df):
     return result_string
 
 
-def get_recommendations_text(suggestions, n_results=10):
-    recommendations = get_recommendations_df(suggestions, n_results)
+def get_recommendations_text(suggestions, n_results=10, similarity=COSINE_SIMILARITY):
+    recommendations = get_recommendations_df(suggestions, n_results, similarity=similarity)
     return recommendations_to_text(recommendations)
     
 if __name__ == "__main__":
     suggestions = {"Pepe": [0, 2960], "Juan": [1945, 6174]} # Pepe: Toy Story 1 y 2. Juan: Peter Pan y Buscando a Nemo
-    get_recommendations_text(suggestions)
+    result = get_recommendations_text(suggestions, similarity=EUCLIDEAN_SIMILARITY)
+    print("\n\n--------\n\n")
+    print(result)
 
 
 

course_project/data_persistence.py

@@ -3,6 +3,7 @@ import os
 
 DATABASE_LOCATION = "course_project/database/"
 MOVIES_BK_FILE_LOCATION = DATABASE_LOCATION + "provisional_database_simulation.pkl"
+MOVIES_BK_FILE_TRANSFORMED_LOCATION = DATABASE_LOCATION + "provisional_database_transformed.pkl"
 
 def save_pickle(df, filename):
     with open(filename, 'wb') as f:
@@ -21,3 +22,12 @@ def get_db():
         return read_pickle(MOVIES_BK_FILE_LOCATION)
     else:
         return None
+
+def save_transformed_db(series_to_store):
+    save_pickle(series_to_store, MOVIES_BK_FILE_TRANSFORMED_LOCATION)
+
+def get_transformed_db():
+    if os.path.exists(MOVIES_BK_FILE_TRANSFORMED_LOCATION):
+        return read_pickle(MOVIES_BK_FILE_TRANSFORMED_LOCATION)
+    else:
+        return None

course_project/embedding.py

@@ -9,10 +9,13 @@ import ollama
 from course_project import data_persistence
 
 DATA_FOLDER_LOCATION = "data/"
-QUESTIONS_FILE_LOCATION = DATA_FOLDER_LOCATION + "train.csv"
-QUESTIONS_WITH_EMBEDDINGS_LOCATION = DATA_FOLDER_LOCATION + "train_with_embeddings.pkl"
 WORD2VEC_FILE_LOCATION = DATA_FOLDER_LOCATION + "glove.6B.100d.txt"
 WORD2VEC_MODEL_FILE_LOCATION = DATA_FOLDER_LOCATION + "glove_model"
+MOVIES_BK_FILE_LOCATION = "course_project/database/provisional_database_simulation.pkl"
+MOVIES_BK_FILE_TRANSFORMED_LOCATION = "course_project/database/provisional_database_transformed.pkl"
+
+COSINE_SIMILARITY = 0
+EUCLIDEAN_SIMILARITY = 1
 
 # ollama pull gemma:7b
 
@@ -93,34 +96,65 @@ def encode_text_series_ollama_method(original_series, file_backup_location=None,
 
 def find_most_similar_coincidences_indexes(n_results, objective_vector, movies_embeddings):
     cosine_sim_scores = cosine_similarity([objective_vector.tolist()], movies_embeddings.tolist())[0].tolist()
-    return [cosine_sim_scores.index(winner_score) for winner_score in sorted(cosine_sim_scores, reverse=True)[:n_results]]
-
-def get_winning_choice_per_person(movie_tested, suggestions):
-    cosine_sim_scores = cosine_similarity([movie_tested.tolist()], suggestions.tolist())[0].tolist()
-    cosine_sim_scores = [score if score<=0.99999 else 0 for score in cosine_sim_scores]
-    winning_score = sorted(cosine_sim_scores, reverse=True)[0]
-    winning_suggestion_index = cosine_sim_scores.index(winning_score)
-    return winning_suggestion_index, winning_score
-
-def find_recommendations(suggestions_dict, movies_embeddings, n_results=10, rating_weight=0):
+    most_similar_indexes = np.argsort(cosine_sim_scores)[-n_results:][::-1]
+    return most_similar_indexes.tolist()
+
+def get_winning_choice_per_person(movie_tested, suggestions, similarity=COSINE_SIMILARITY):
+    if similarity==COSINE_SIMILARITY:
+        scores = np.array(cosine_similarity([movie_tested.tolist()], suggestions.tolist())[0].tolist())
+        scores[scores >= 0.99999] = 0
+        winning_suggestion_index = np.argmax(scores)
+        winning_score = scores[winning_suggestion_index]
+        return winning_suggestion_index, winning_score
+    elif similarity==EUCLIDEAN_SIMILARITY:
+        distances = np.linalg.norm(np.array(suggestions.tolist()) - movie_tested, axis=1)
+        scores = np.exp(-0.005*distances)
+        scores[scores >= 0.99999] = 0
+        winning_suggestion_index = np.argmax(scores)
+        winning_score = scores[winning_suggestion_index]
+        return winning_suggestion_index, winning_score
+    else:
+        raise ValueError('Wrong similarity value')
+
+def normalize_to_minus_one_and_one(arr):
+    min_val = np.min(arr)
+    max_val = np.max(arr)
+    # Evitamos la división por cero en caso de que todos los valores sean iguales
+    if min_val == max_val:
+        return np.zeros_like(arr)
+    normalized_arr = 2 * (arr - min_val) / (max_val - min_val) - 1
+    return normalized_arr
+
+def linearly_transform_embeddings():
+    movie_data = data_persistence.read_pickle(MOVIES_BK_FILE_LOCATION)
+    embeddings = movie_data["vector"]
+    embeddings = np.array(embeddings)
+    for embedding in embeddings:
+        # print(embedding)
+        embedding = normalize_to_minus_one_and_one(np.array(embedding))
+        embedding = pd.Series(embedding)
+    embeddings = pd.Series(embeddings)
+    movie_data["vector"] = embeddings
+    data_persistence.save_pickle(movie_data, MOVIES_BK_FILE_TRANSFORMED_LOCATION)
+
+def find_recommendations(suggestions_dict, movies_embeddings, n_results=10, rating_weight=0, similarity=COSINE_SIMILARITY):
     scores = []
     suggestions_considered = []
     for movie_tested in movies_embeddings:
         movie_tested_score = 0
         movie_tested_suggestions_considered = {}
         for person in suggestions_dict:
-            winning_suggestion_index, winning_score = get_winning_choice_per_person(movie_tested, movies_embeddings.loc[suggestions_dict[person]])
+            winning_suggestion_index, winning_score = get_winning_choice_per_person(movie_tested, movies_embeddings.loc[suggestions_dict[person]], similarity=similarity)
             movie_tested_score += winning_score
             movie_tested_suggestions_considered[person] = suggestions_dict[person][winning_suggestion_index]
         movie_tested_score /= len(suggestions_dict)
         movie_tested_score += rating_weight
         scores.append(movie_tested_score)
         suggestions_considered.append(movie_tested_suggestions_considered)
-
-    recommended_movies_indexes = [scores.index(winner_score) for winner_score in sorted(scores, reverse=True)[:n_results]]
-    recommendations = []
-    for recommended_movie_index in recommended_movies_indexes:
-        recommendation = {"movie_index": recommended_movie_index, "suggestions_considered": suggestions_considered[recommended_movie_index]}
-        recommendations.append(recommendation)
+    
+    recommended_movies_indexes = np.argsort(scores)[-n_results:][::-1]
+    recommendations = [
+        {"movie_index": idx, "suggestions_considered": suggestions_considered[idx]}
+        for idx in recommended_movies_indexes
+    ]
     return recommendations
-