update app.py

app.py

@@ -5,39 +5,51 @@ import numpy as np
 from sklearn.preprocessing import KBinsDiscretizer, StandardScaler, OneHotEncoder
 from sklearn.feature_extraction.text import HashingVectorizer
 from collections import Counter
+import joblib
 import freesound
 import gensim.downloader as api
 
-
+# -------- FreeSound API --------
 client = freesound.FreesoundClient()
 client.set_token("zE9NjEOgUMzH9K7mjiGBaPJiNwJLjSM53LevarRK", "token")
 
-# Répertoire dataset
 dataset_dir = "dataset_audio"
 os.makedirs(dataset_dir, exist_ok=True)
 
-# Liste des métadonnées importantes
-metadata_cols = ["name", "num_ratings", "tags", "username",
-    "description", "created", "license", "num_downloads", "channels",
-    "filesize","num_comments", "category_is_user_provided", "duration", "avg_rating",
-    "category", "subcategory", "type","samplerate , beat_count ,amplitude_peak_ratio "
-]
+# -------- Charger les objets sauvegardés --------
+# Music
+scaler_samplerate_music = joblib.load("music/scaler_samplerate.joblib")
+scaler_age_days_music = joblib.load("music/scaler_music_age_days_log.joblib")
+username_freq_music = joblib.load("music/username_freq_dict_music.joblib")
+est_num_downloads_music = joblib.load("music/est_num_downloads_music.joblib")
+avg_rating_transformer_music = joblib.load("music/avg_rating_transformer_music.joblib")
+music_subcategory_cols = joblib.load("music/music_subcategory_cols.joblib")
+music_onehot_cols = joblib.load("music/music_onehot_cols.joblib")
+
+# EffectSound
+scaler_samplerate_effect = joblib.load("effectSound/scaler_samplerate.joblib")
+scaler_age_days_effect = joblib.load("effectSound/scaler_effectSound_age_days_log.joblib")
+username_freq_effect = joblib.load("effectSound/username_freq_dict_effectSound.joblib")
+est_num_downloads_effect = joblib.load("effectSound/est_num_downloads_effectSound.joblib")
+avg_rating_transformer_effect = joblib.load("effectSound/avg_rating_transformer_effectSound.joblib")
+effect_subcategory_cols = joblib.load("effectSound/effectSound_subcategory_cols.joblib")
+effect_onehot_cols = joblib.load("effectSound/effectSound_onehot_cols.joblib")
+
+# GloVe pour description
+glove_model = api.load("glove-wiki-gigaword-100")
 
+# -------- Fonctions --------
 def fetch_sound_metadata(sound_url):
-    # Extraire l'ID FreeSound de l'URL
+    """Télécharge les métadonnées du son FreeSound"""
     sound_id = int(sound_url.rstrip("/").split("/")[-1])
     sound = client.get_sound(sound_id)
-
     file_name = f"{sound.name.replace(' ', '_')}.mp3"
     file_path = os.path.join(dataset_dir, file_name)
-
-    # Télécharger le preview
     try:
         sound.retrieve_preview(dataset_dir, file_name)
     except Exception as e:
-        print(f"Erreur téléchargement {file_name} : {e}")
+        print(f"Erreur téléchargement {file_name}: {e}")
         file_path = None
-
     data = {
         "file_path": file_path,
         "name": sound.name,
@@ -57,291 +69,119 @@ def fetch_sound_metadata(sound_url):
         "category": getattr(sound, "category", "Unknown"),
         "subcategory": getattr(sound, "subcategory", "Other"),
         "type": getattr(sound, "type", ""),
-        "samplerate": getattr(sound, "samplerate", 0),
-        "amplitude_peak_ratio":getattr(sound, "amplitude_peak_ratio", ""),
-         "beat_count":getattr(sound, "beat_count", "")
+        "samplerate": getattr(sound, "samplerate", 0)
     }
     return pd.DataFrame([data])
 
-def preprocess_subcategory_ohe(df, seuil=2):
-    df = df.copy()
-    df["subcategory"] = df["subcategory"].fillna("Other")
-    counts = df["subcategory"].value_counts(normalize=True) * 100
-    rare_subs = counts[counts < seuil].index
-    df["subcategory"] = df["subcategory"].apply(lambda x: "Other" if x in rare_subs else x)
-
-    ohe = OneHotEncoder(sparse_output=False)
-    subcat_ohe = ohe.fit_transform(df[["subcategory"]])
-    subcat_df = pd.DataFrame(
-        subcat_ohe,
-        columns=[f"subcategory_{c}" for c in ohe.categories_[0]],
-        index=df.index
-    )
-    return pd.concat([df, subcat_df], axis=1)
-
-
-def preprocess_tags_ohe(df, seuil=1.0):
-    df = df.copy()
-    df["tags"] = df["tags"].fillna("")
-    df["tags_list"] = df["tags"].str.lower().str.split(",")
-
-    all_tags = [t.strip() for sub in df["tags_list"] for t in sub if t.strip()]
-    counts = Counter(all_tags)
-    total = len(df)
-
-    frequent_tags = {
-        tag for tag, cnt in counts.items()
-        if cnt / total * 100 >= seuil
-    }
-
-    def filter_tags(tags):
-        return list(set([t if t in frequent_tags else "Other" for t in tags]))
-
-    df["tags_list"] = df["tags_list"].apply(filter_tags)
-
-    tags_for_ohe = df["tags_list"].apply(lambda x: ";".join(x)).to_numpy().reshape(-1, 1)
-    ohe = OneHotEncoder(sparse_output=False)
-    tags_ohe = ohe.fit_transform(tags_for_ohe)
-
-    cols = [f"tag_{c}" for c in ohe.categories_[0]]
-    df_tags = pd.DataFrame(tags_ohe, columns=cols, index=df.index)
-
-    return pd.concat([df, df_tags], axis=1)
-
-
-def preprocess_targets(df):
-    # num_downloads -> discretisation 3 classes
-    X = df["num_downloads"].to_numpy().reshape(-1,1)
-    est = KBinsDiscretizer(n_bins=3, encode="ordinal", strategy="quantile")
-    df["num_downloads_class"] = est.fit_transform(X).astype(int)
-
-    # avg_rating -> discretisation en 4 classes
-    mask_non_zero = df["avg_rating"] != 0
-    X_non_zero = df.loc[mask_non_zero, "avg_rating"].to_numpy().reshape(-1,1)
-    est = KBinsDiscretizer(n_bins=3, encode="ordinal", strategy="quantile")
-    df["avg_rating_class"] = 0
-    df.loc[mask_non_zero, "avg_rating_class"] = est.fit_transform(X_non_zero).flatten().astype(int) + 1
-    df["avg_rating"] = df["avg_rating_class"]
-    df.drop(columns=["avg_rating_class"], inplace=True)
-
-    return df
-
-
 def description_to_vec(text, model, dim=100):
     if not text:
         return np.zeros(dim)
     words = text.lower().split()
     vecs = [model[w] for w in words if w in model]
-    return np.mean(vecs, axis=0) if vecs else np.zeros(dim)
-
-
-def preprocess_features(df):
-    df = df.copy()
-
-    # Colonnes booléennes
-    df["category_is_user_provided"] = df["category_is_user_provided"].astype(int)
-
-    # Colonnes catégorielles -> one-hot
-    cat_cols = ["license", "category", "type"]
-    df[cat_cols] = df[cat_cols].fillna("Unknown")
-    df = pd.get_dummies(df, columns=cat_cols, drop_first=False)
-
-    # username -> frequency encoding
-    user_freq = df["username"].value_counts(normalize=True)
-    df["username_freq"] = df["username"].map(user_freq)
-    df.drop(columns=["username"], inplace=True)
-
-    # subcategory -> one-hot, rare <2% regroupé
-    df["subcategory"] = df["subcategory"].fillna("Other")
-    counts = df["subcategory"].value_counts(normalize=True)*100
-    rare_subs = counts[counts<2].index
-    df["subcategory"] = df["subcategory"].apply(lambda x: "Other" if x in rare_subs else x)
-    ohe = OneHotEncoder(sparse_output=False)
-    subcat_ohe = ohe.fit_transform(df[["subcategory"]])
-    subcat_df = pd.DataFrame(subcat_ohe, columns=[f"subcategory_{c}" for c in ohe.categories_[0]], index=df.index)
-    df = pd.concat([df, subcat_df], axis=1)
-    df.drop(columns=["subcategory"], inplace=True)
-
-    # Colonnes numériques -> log1p + standard scaler
-    numeric_cols = ["num_ratings", "filesize", "duration", "samplerate"]
-    for col in numeric_cols:
-        df[col] = np.log1p(df[col])
-    scaler = StandardScaler()
-    df[numeric_cols] = scaler.fit_transform(df[numeric_cols])
-
-    # Description -> vecteur GloVe 100 dim
-    glove_model = api.load("glove-wiki-gigaword-100")
-    def description_to_vec(text, model):
-        if not text: return np.zeros(100)
-        words = text.lower().split()
-        vecs = [model[w] for w in words if w in model]
-        return np.mean(vecs, axis=0) if vecs else np.zeros(100)
-    desc_vecs = np.vstack(df['description'].fillna('').apply(lambda x: description_to_vec(x, glove_model)))
-    desc_cols = [f'description_glove_{i}' for i in range(desc_vecs.shape[1])]
-    df[desc_cols] = pd.DataFrame(desc_vecs, columns=desc_cols, index=df.index)
-    df.drop(columns=["description"], inplace=True)
-
-    return df
+    if len(vecs) == 0:
+        return np.zeros(dim)
+    return np.mean(vecs, axis=0)
 
-def preprocess_features_full(df):
+def preprocess_sound(df):
+    """Applique le preprocessing complet selon duration pour choisir music ou effectSound"""
     df = df.copy()
+    dur = df["duration"].iloc[0]
     
-    # 1. Colonnes booléennes
-    df["category_is_user_provided"] = df["category_is_user_provided"].astype(int)
-
-    # 2. Colonnes catégorielles → one-hot
-    onehot_cols = ["license", "category", "type"]
-    df_encoded = pd.get_dummies(df[onehot_cols], drop_first=False)
-    df.drop(columns=onehot_cols, inplace=True)
-    df = pd.concat([df, df_encoded], axis=1)
-
-    # 3. Subcategory
-    df = preprocess_subcategory_ohe(df, seuil=2)
-    df.drop(columns=["subcategory"], inplace=True)
-
-    # 4. Colonnes numériques
-    numeric_cols = ["num_ratings", "num_comments", "filesize", "duration"]
-    for col in numeric_cols:
+    if 0.5 <= dur <= 3:
+        dataset_type = "effectSound"
+        scaler_samplerate = scaler_samplerate_effect
+        scaler_age = scaler_age_days_effect
+        username_freq = username_freq_effect
+        est_num_downloads = est_num_downloads_effect
+        avg_rating_transformer = avg_rating_transformer_effect
+        subcat_cols = effect_subcategory_cols
+        onehot_cols = effect_onehot_cols
+    elif 10 <= dur <= 60:
+        dataset_type = "music"
+        scaler_samplerate = scaler_samplerate_music
+        scaler_age = scaler_age_days_music
+        username_freq = username_freq_music
+        est_num_downloads = est_num_downloads_music
+        avg_rating_transformer = avg_rating_transformer_music
+        subcat_cols = music_subcategory_cols
+        onehot_cols = music_onehot_cols
+    else:
+        return f"❌ Son trop court ou trop long ({dur} sec)"
+
+    # ----------------- Features -----------------
+    # Category bool
+    df["category_is_user_provided"] = int(df["category_is_user_provided"])
+    
+    # Username frequency
+    df["username_freq"] = df["username"].map(username_freq).fillna(0)
+    
+    # Numeric features
+    for col in ["num_ratings", "num_comments", "filesize", "duration"]:
         df[col] = np.log1p(df[col])
-    scaler = StandardScaler()
-    df["samplerate"] = scaler.fit_transform(df[["samplerate"]])
-
-    # 5. age_days
+    df["samplerate"] = scaler_samplerate.transform(df[["samplerate"]])
+    
+    # Age_days
     df["created"] = pd.to_datetime(df["created"], errors="coerce")
     df["age_days"] = (pd.Timestamp.now() - df["created"]).dt.days
     df["age_days_log"] = np.log1p(df["age_days"])
-    df["age_days_log_scaled"] = StandardScaler().fit_transform(df[["age_days_log"]])
-    df.drop(columns=["created", "age_days", "age_days_log"], inplace=True)
-
-    # 6. username → frequency encoding
-    user_freq = df["username"].value_counts(normalize=True)
-    df["username_freq"] = df["username"].map(user_freq)
-    df.drop(columns=["username"], inplace=True)
-
-    # 7. Tags
-    df = preprocess_tags_ohe(df, seuil=1.0)
-    df.drop(columns=["tags", "tags_list"], inplace=True)
-
-    # 8. Name
-    df["name_clean"] = df["name"].str.lower().str.rsplit(".", n=1).str[0]
+    df["age_days_log_scaled"] = scaler_age.transform(df[["age_days_log"]])
+    
+    # num_downloads
+    df["num_downloads_class"] = est_num_downloads.transform(df[["num_downloads"]])
+    
+    # avg_rating
+    df["avg_rating"] = avg_rating_transformer.transform(df["avg_rating"].to_numpy())
+    
+    # Subcategory
+    for col in subcat_cols:
+        df[col] = 0  # pour gradio, on va juste créer les colonnes
+    
+    # One-hot
+    for col in onehot_cols:
+        df[col] = 0
+    
+    # Tags
+    df["tags_list"] = df["tags"].fillna("").str.lower().str.split(",")
+    # One-Hot Encoding tags à la volée
+    all_tags = [t.strip() for sub in df["tags_list"] for t in sub if t.strip() != ""]
+    frequent_tags = set(all_tags)  # simplifié, car threshold appliqué dans Colab déjà
+    for tag in frequent_tags:
+        df[f"tag_{tag.replace(' ','_')}"] = 1
+    df.drop(columns=["tags","tags_list"], inplace=True)
+    
+    # Name
+    df["name_clean"] = df["name"].str.lower().str.rsplit(".",1).str[0]
     vectorizer = HashingVectorizer(n_features=8, alternate_sign=False, norm=None)
     name_vec = vectorizer.transform(df["name_clean"])
-    name_vec_df = pd.DataFrame(name_vec.toarray(), columns=[f"name_vec_{i}" for i in range(8)], index=df.index)
-    df = pd.concat([df, name_vec_df], axis=1)
-    df.drop(columns=["name", "name_clean"], inplace=True)
-
-    # 9. Description → GloVe 100
-    glove_model = api.load("glove-wiki-gigaword-100")
-    desc_vecs = np.vstack(df['description'].fillna('').apply(lambda x: description_to_vec(x, glove_model)))
-    desc_cols = [f'description_glove_{i}' for i in range(desc_vecs.shape[1])]
-    df[desc_cols] = pd.DataFrame(desc_vecs, columns=desc_cols, index=df.index)
+    for i in range(8):
+        df[f"name_vec_{i}"] = name_vec.toarray()[0][i]
+    df.drop(columns=["name","name_clean"], inplace=True)
+    
+    # Description
+    desc_vec = description_to_vec(df["description"].iloc[0], glove_model)
+    for i in range(100):
+        df[f"description_glove_{i}"] = desc_vec[i]
     df.drop(columns=["description"], inplace=True)
-
-    # 10. Supprimer colonnes inutiles restantes
-    cols_to_remove = ["file_path","previews","similar_sounds","comments","geotag","bitrate","bitdepth","is_remix","was_remixed"]
-    df = df.drop(columns=[c for c in cols_to_remove if c in df.columns], errors="ignore")
-
-    return df
-
-
-def preprocess_features_step1(df):
-    df = df.copy()
-
-    # 1. Bool → int
-    df["category_is_user_provided"] = df["category_is_user_provided"].astype(int)
-
-    # 2. Catégorielles simples → one-hot
-    cat_cols = ["license", "category", "type"]
-    df[cat_cols] = df[cat_cols].fillna("Unknown")
-    df = pd.get_dummies(df, columns=cat_cols, drop_first=False)
-
-    # 3. username → frequency encoding
-    user_freq = df["username"].value_counts(normalize=True)
-    df["username_freq"] = df["username"].map(user_freq)
-    df.drop(columns=["username"], inplace=True)
-
-    # 4. Numériques simples
-    num_cols = ["num_ratings", "filesize", "duration"]
-    for col in num_cols:
-        df[col] = np.log1p(df[col])
-
-    scaler = StandardScaler()
-    df[["samplerate"]] = scaler.fit_transform(df[["samplerate"]])
-
+    
     return df
 
-
-# -------- Fonctions --------
-
+# -------- Gradio --------
 def predict_with_metadata(url):
-    if url.strip() == "":
-        return " Veuillez entrer une URL FreeSound."
-
-    df = fetch_sound_metadata(url)
-    df = preprocess_targets(df)
-    #df = preprocess_features_step1(df)
-
-    #df_raw = df.copy()
-    #df = preprocess_targets(df)
-    #df = preprocess_features_full(df)  # <-- version complète
-    
-    # Pour l'affichage
-    #avg_rating = df["avg_rating"].iloc[0]
-    #num_downloads_class = df["num_downloads_class"].iloc[0]
-    #total_columns = df.shape[1]
-
-    #raw_display = "\n".join(
-     #       [f"{col} : {df_raw[col].iloc[0]}" for col in df_raw.columns]
-      #  )
-    print("=== COLONNES AVANT PRÉTRAITEMENT ===")
-    print(df.columns.tolist())
-    print(df.dtypes)
-    return (
-             "OK – FEATURES STEP 1\n"
-            f"Nombre de colonnes : {df.shape[1]}\n\n"
-            f"{df.dtypes}"
-            f"{df.columns.dtype}"
-            )
-            
-
-def predict_with_audio(url):
     if url.strip() == "":
         return "❌ Veuillez entrer une URL FreeSound."
     
-    # PLUS TARD :
-    # - télécharger audio
-    # - extraire features acoustiques
-    
-    return "🎵 Résultat (audio) : SON NON POPULAIRE (exemple)"
-
-
-# -------- Interface Gradio --------
+    df = fetch_sound_metadata(url)
+    df_processed = preprocess_sound(df)
+    return df_processed.to_string()
 
 with gr.Blocks(title="FreeSound Popularity Detector") as demo:
     gr.Markdown("# 🎧 FreeSound Popularity Detector")
-    gr.Markdown("Collez l'URL d'un son FreeSound et choisissez la méthode d'analyse.")
-
-    url_input = gr.Textbox(
-        label="URL du son FreeSound",
-        placeholder="https://freesound.org/people/..."
-    )
-
-    with gr.Row():
-        btn_meta = gr.Button("📊 Prédire via métadonnées")
-        btn_audio = gr.Button("🎼 Prédire via données acoustiques")
+    gr.Markdown("Collez l'URL d'un son FreeSound et le preprocessing complet sera appliqué automatiquement.")
 
+    url_input = gr.Textbox(label="URL du son FreeSound")
+    btn_meta = gr.Button("📊 Prétraiter et afficher features")
     output = gr.Textbox(label="Résultat")
 
-    btn_meta.click(
-        fn=predict_with_metadata,
-        inputs=url_input,
-        outputs=output
-    )
-
-    btn_audio.click(
-        fn=predict_with_audio,
-        inputs=url_input,
-        outputs=output
-    )
+    btn_meta.click(fn=predict_with_metadata, inputs=url_input, outputs=output)
 
 demo.launch()

effectSound/scaler_effectSound_age_days_log.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:332ee96e7bca4c412bc0d5ac20c0876d5bf8304142d4fd57d4d5524e03228e61
+size 895

effectSound/username_freq_dict_effectSound.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:457517d900b3c05061f398d37b00f8087ae9edb1a4776c7cbc2fc77fa60a4036
+size 209269

music/scaler_music_age_days_log.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1941f5f48e21243a939080d9d7a1cedc677e2b0b813a451a50f64d00ce149588
+size 895

music/username_freq_dict_music.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6ca2078e0e0c5c0d5f871362bba1e787c6860fb547dd1f9f3c4f0f3c366b447
+size 214933