Added some optimization

app.py

@@ -10,6 +10,7 @@ from GameRecommender import *
 import gc
 from sklearn.model_selection import train_test_split
 from huggingface_hub import snapshot_download
+from sklearn.preprocessing import MultiLabelBinarizer,LabelEncoder,MinMaxScaler
 
 DATASETS = {
     "converted": "converted.csv",
@@ -75,7 +76,9 @@ df_review_raw = REVIEWS_DS['train'].to_pandas()
 df_review_trimmed = TRIMMED_REVIEWS_DS['train'].to_pandas()
 df_user_pref = USER_PREF_DS['train'].to_pandas()
 available_names = df_games[df_games['app_id'].astype(str).isin(selectable_app_ids)]['Name'].tolist()
-
+min_word=20
+df_review_trimmed_filtered = df_review_trimmed[df_review_trimmed['cleaned_review'].apply(lambda x: len(str(x).split()) >=min_word)].reset_index(drop=True)
+                
 def extract_year(date_str):
     if isinstance(date_str, str) and len(date_str) >= 4:
         year_str = date_str[-4:]
@@ -338,7 +341,7 @@ df_games_raw['Publishers'] = df_games_raw['Publishers'].fillna('')
 df_games_raw.to_csv('Cleaned_games.csv',index=False)
 """)
                 h2('Games Data Cleaned')
-                gr.DataFrame(df_games.head(20))
+                gr.Dataframe(df_games.head(20))
                 
                 h2('2.2. Review Preprocessing')
                 Dataset(df_review_raw,'Review Data Raw',REVIEWS_DATAPATH)
@@ -413,8 +416,6 @@ df = df[['steamid','app_id','voted_up','cleaned_review']]
 df.to_csv('Cleaned_Dataframe.csv',index=False)
 """)
                 Dataset(df_review_trimmed,'Cleaned Review',source=TRIMMED_REVIEW_DATAPATH,key='trimmed_review')
-                min_word=20
-                df_review_trimmed_filtered = df_review_trimmed[df_review_trimmed['cleaned_review'].apply(lambda x: len(str(x).split()) >=min_word)].reset_index(drop=True)
                 code_cell("""
 min_word = 20
 df = df[df['cleaned_review'].apply(lambda x: len(str(x).split()) >=min_word)].reset_index(drop=True)
@@ -528,20 +529,23 @@ df_liked = df_liked.drop_duplicates(subset=['steamid', 'app_id'])
                 code_cell("""
 vectorizer = TfidfVectorizer(max_df=0.7,min_df=3,stop_words=None,ngram_range=(1,2))
 review_app_id_encoder = LabelEncoder()""")
-                train_df,df_temp = train_test_split(sampled,test_size=0.2,random_state=SEED,stratify=sampled['app_id'])
-                test_df,val_df = train_test_split(df_temp,test_size=0.5,random_state=SEED,stratify=df_temp['app_id'])
-                del df_temp
-                gc.collect()
-                code_cell("""
-train_df,df_temp = train_test_split(sampled,test_size=0.2,random_state=SEED,stratify=sampled['app_id'])
-test_df,val_df = train_test_split(df_temp,test_size=0.5,random_state=SEED,stratify=df_temp['app_id'])
-""")
-                p(f"""
+                def get_data_split():
+                    train_df,df_temp = train_test_split(sampled,test_size=0.2,random_state=SEED,stratify=sampled['app_id'])
+                    test_df,val_df = train_test_split(df_temp,test_size=0.5,random_state=SEED,stratify=df_temp['app_id'])
+                    del df_temp
+                    gc.collect()
+                    p(f"""
 Training   : {train_df.shape}
 Testing    : {test_df.shape}
 Validation : {val_df.shape}
 """)            
                 code_cell("""
+train_df,df_temp = train_test_split(sampled,test_size=0.2,random_state=SEED,stratify=sampled['app_id'])
+test_df,val_df = train_test_split(df_temp,test_size=0.5,random_state=SEED,stratify=df_temp['app_id'])
+""")
+                btn = gr.Button("View data split size :")
+                btn.click(fn=get_data_split())
+                code_cell("""
 X_train = vectorizer.fit_transform(train_df['cleaned_review'])
 y_train = review_app_id_encoder.fit_transform(train_df['app_id'])
 X_test = vectorizer.transform(test_df['cleaned_review'])
@@ -868,16 +872,6 @@ df = col_to_list(df,'Categories')
 df = apply_price_range_labels(df,price_labels,price_bins)
                           """)
                 Dataset(df_games,"The game dataset",GAMES_DATAPATH)
-                df_games_temp = df_games
-                df_games_temp = col_to_list(df_games_temp,'Genres')
-                df_games_temp = col_to_list(df_games_temp,'Categories')
-                df_games_temp = apply_price_range_labels(df_games_temp,price_ranges_labels,price_bins)
-                df_games_temp['Year_Release'] = df_games_temp['Release date'].apply(extract_year)
-                df_games_temp['Game score'] = np.where(
-                    (df_games_temp['Positive'] + df_games_temp['Negative']) == 0,
-                    0,
-                    (df_games_temp['Positive'] / (df_games_temp['Positive'] + df_games_temp['Negative'])) * 100
-                )
                 
                 code_cell("""
 def extract_year(date_str):
@@ -893,25 +887,36 @@ df['Game score'] = np.where(
     0,
     (df['Positive'] / (df['Positive'] + df['Negative'])) * 100
 )""")
-                from sklearn.preprocessing import MultiLabelBinarizer,LabelEncoder,MinMaxScaler
-                genre_mlb = MultiLabelBinarizer()
-                genre_mlb = genre_mlb.fit(df_games_temp['Genres'])
-                categories_mlb = MultiLabelBinarizer()
-                categories_mlb = categories_mlb.fit(df_games_temp['Categories'])
-                price_range_le = model.game_content_recommeder.price_range_encoder
-                scaler = MinMaxScaler()
-                scaler = scaler.fit(df_games_temp[['Year_Release','Average playtime forever','Game score','DLC count']].values)
-                app_id_le = LabelEncoder()
-                app_id_le = app_id_le.fit(df_games_temp['app_id'])
-                numerical_col =['Year_Release','Average playtime forever','Game score','DLC count']
-                
-                genre_matrix = genre_mlb.transform(df_games_temp['Genres'])
-                genre_df = pd.DataFrame(genre_matrix, columns=genre_mlb.classes_, index=df_games_temp.index)
-                categories_matrix = categories_mlb.transform(df_games_temp['Categories'])
-                categories_df = pd.DataFrame(categories_matrix,columns=categories_mlb.classes_,index=df_games_temp.index)
-                game_df = pd.concat([df_games_temp[['app_id','Price_range']+numerical_col],genre_df,categories_df],axis=1)
-                game_df['Price_range'] = price_range_le.transform(game_df['Price_range'])
-                game_df[numerical_col] = scaler.transform(game_df[numerical_col].values)
+                def game_df_create():
+                    df_games_temp = df_games
+                    df_games_temp = col_to_list(df_games_temp,'Genres')
+                    df_games_temp = col_to_list(df_games_temp,'Categories')
+                    df_games_temp = apply_price_range_labels(df_games_temp,price_ranges_labels,price_bins)
+                    df_games_temp['Year_Release'] = df_games_temp['Release date'].apply(extract_year)
+                    df_games_temp['Game score'] = np.where(
+                        (df_games_temp['Positive'] + df_games_temp['Negative']) == 0,
+                        0,
+                        (df_games_temp['Positive'] / (df_games_temp['Positive'] + df_games_temp['Negative'])) * 100
+                    )
+                    genre_mlb = MultiLabelBinarizer()
+                    genre_mlb = genre_mlb.fit(df_games_temp['Genres'])
+                    categories_mlb = MultiLabelBinarizer()
+                    categories_mlb = categories_mlb.fit(df_games_temp['Categories'])
+                    price_range_le = model.game_content_recommeder.price_range_encoder
+                    scaler = MinMaxScaler()
+                    scaler = scaler.fit(df_games_temp[['Year_Release','Average playtime forever','Game score','DLC count']].values)
+                    app_id_le = LabelEncoder()
+                    app_id_le = app_id_le.fit(df_games_temp['app_id'])
+                    numerical_col =['Year_Release','Average playtime forever','Game score','DLC count']
+                    
+                    genre_matrix = genre_mlb.transform(df_games_temp['Genres'])
+                    genre_df = pd.DataFrame(genre_matrix, columns=genre_mlb.classes_, index=df_games_temp.index)
+                    categories_matrix = categories_mlb.transform(df_games_temp['Categories'])
+                    categories_df = pd.DataFrame(categories_matrix,columns=categories_mlb.classes_,index=df_games_temp.index)
+                    game_df = pd.concat([df_games_temp[['app_id','Price_range']+numerical_col],genre_df,categories_df],axis=1)
+                    game_df['Price_range'] = price_range_le.transform(game_df['Price_range'])
+                    game_df[numerical_col] = scaler.transform(game_df[numerical_col].values)
+                    return game_df.head(10)
                 code_cell("""
 from sklearn.preprocessing import MultiLabelBinarizer,LabelEncoder,MinMaxScaler
 genre_mlb = MultiLabelBinarizer()
@@ -932,7 +937,10 @@ genre_df = pd.DataFrame(genre_matrix, columns=genre_mlb.classes_, index=df.index
 categories_matrix = categories_mlb.transform(df['Categories'])
 categories_df = pd.DataFrame(categories_matrix,columns=categories_mlb.classes_,index=df.index)
 game_df = pd.concat([df[['app_id','Price_range']+numerical_col],genre_df,categories_df],axis=1)""")
-                gr.Dataframe(game_df.head(10))
+
+                btn = gr.Button("Run game_df preprocess")
+                output_game_df = gr.Dataframe()
+                btn.click(fn=game_df_create, inputs=None, outputs=output_game_df)
                 code_cell("""
 from sklearn.neighbors import KNeighborsClassifier
 X = game_df.loc[:,['Year_Release','Average playtime forever','Game score','DLC count','Price_range']+ list(genre_mlb.classes_) + list(categories_mlb.classes_)]