testtest5

app.py

@@ -1,5 +1,4 @@
 import os
-import glob
 import tempfile
 import numpy as np
 import pandas as pd
@@ -9,72 +8,52 @@ import joblib
 import soundfile as sf
 from pydub import AudioSegment
 import opensmile
+
 import freesound
 import xgboost as xgb
+from sklearn.feature_extraction.text import HashingVectorizer
 
 
-# =========================
+# ============================================================
 # CONFIG
-# =========================
+# ============================================================
 MIN_EFFECT, MAX_EFFECT = 0.5, 3.0
 MIN_MUSIC, MAX_MUSIC = 10.0, 60.0
 SR_TARGET = 16000
 
+# HF Space Secret: FREESOUND_TOKEN
 FREESOUND_TOKEN = os.getenv("FREESOUND_TOKEN", "").strip()
 BASE_DIR = os.path.dirname(os.path.abspath(__file__))
 
-
-# =========================
-# Helpers fichiers
-# =========================
 def p(*parts):
     return os.path.join(BASE_DIR, *parts)
 
-def list_local_files():
-    files = []
-    for root, _, fnames in os.walk(BASE_DIR):
-        for f in fnames:
-            if f.lower().endswith((".pkl", ".joblib", ".json", ".bin", ".txt")):
-                rel = os.path.relpath(os.path.join(root, f), BASE_DIR)
-                files.append(rel)
-    return sorted(files)
-
-def exists(rel_path: str) -> bool:
-    return os.path.exists(p(rel_path))
-
-def load_joblib_any(candidates):
-    """
-    Essaie une liste de chemins relatifs (ou patterns glob).
-    Retourne (obj, chosen_path) ou (None, None).
-    """
-    for c in candidates:
-        if any(ch in c for ch in ["*", "?", "["]):
-            matches = sorted(glob.glob(p(c)))
-            if not matches:
-                continue
-            chosen = matches[0]
-            try:
-                obj = joblib.load(chosen)
-                return obj, os.path.relpath(chosen, BASE_DIR)
-            except Exception:
-                continue
-        else:
-            full = p(c)
-            if os.path.exists(full):
-                try:
-                    obj = joblib.load(full)
-                    return obj, c
-                except Exception:
-                    continue
-    return None, None
+def exists(relpath: str) -> bool:
+    return os.path.exists(p(relpath))
 
+def load_local(relpath: str):
+    full = p(relpath)
+    if not os.path.exists(full):
+        raise FileNotFoundError(f"Fichier introuvable: {relpath}")
+    return joblib.load(full)
 
-# =========================
+def safe_float(v):
+    try:
+        return float(v)
+    except Exception:
+        return 0.0
+
+def parse_sound_id(url: str) -> int:
+    return int(url.rstrip("/").split("/")[-1])
+
+
+# ============================================================
 # UI helpers
-# =========================
+# ============================================================
 CSS = """
 .card { border: 1px solid #e5e7eb; background: #ffffff; padding: 16px; border-radius: 16px; }
 .card-error{ border-color: #fca5a5; background: #fff1f2; }
+.card-warn{ border-color: #fcd34d; background: #fffbeb; }
 .card-title{ font-weight: 950; margin-bottom: 8px; }
 .badges{ display:flex; gap:10px; flex-wrap:wrap; margin-bottom:12px; }
 .badge{ padding:6px 10px; border-radius:999px; font-weight:900; font-size: 13px; border: 1px solid #e5e7eb; }
@@ -85,19 +64,30 @@ CSS = """
 .box-title{ font-weight:900; margin-bottom:4px; }
 .box-value{ font-size:18px; font-weight:800; }
 .hint{ margin-top:10px; color:#6b7280; font-size:12px; }
-pre{ white-space:pre-wrap; }
 #header-title { font-size: 28px; font-weight: 950; margin-bottom: 6px; }
 #header-sub { color:#6b7280; margin-top:0px; line-height:1.45; }
+pre{ white-space:pre-wrap; }
 """
 
-def html_error(title, body_html):
+def html_box(title, body, kind=""):
+    cls = "card"
+    if kind == "error":
+        cls += " card-error"
+    elif kind == "warn":
+        cls += " card-warn"
     return f"""
-<div class="card card-error">
-  <div class="card-title">❌ {title}</div>
-  <div>{body_html}</div>
+<div class="{cls}">
+  <div class="card-title">{title}</div>
+  <div>{body}</div>
 </div>
 """.strip()
 
+def html_error(title, body_html):
+    return html_box(f"❌ {title}", body_html, kind="error")
+
+def html_warn(title, body_html):
+    return html_box(f"⚠️ {title}", body_html, kind="warn")
+
 def html_result(badge_text, duration, rating_text, downloads_text, extra_html=""):
     return f"""
 <div class="card">
@@ -122,7 +112,7 @@ def html_result(badge_text, duration, rating_text, downloads_text, extra_html=""
 
 def interpret_results(avg_class: int, dl_class: int) -> str:
     if avg_class == 0:
-        return "ℹ️ <b>Interprétation</b> :<br>Aucune évaluation possible (rating manquant)."
+        return "ℹ️ <b>Interprétation</b> :<br>Aucune évaluation possible (rating manquant / indisponible)."
 
     if avg_class == 3 and dl_class == 2:
         potentiel, detail = "très fort", "contenu de haute qualité et très populaire."
@@ -159,30 +149,83 @@ def avg_label_to_class(avg_label: str) -> int:
         return 1
     return 0
 
-def safe_float(v):
-    try:
-        return float(v)
-    except Exception:
-        return 0.0
-
-def parse_sound_id(url: str):
-    return int(url.rstrip("/").split("/")[-1])
-
 
-# =========================
+# ============================================================
 # FreeSound client
-# =========================
+# ============================================================
 def get_fs_client():
     if not FREESOUND_TOKEN:
-        raise RuntimeError("Token FreeSound manquant. Ajoute le secret FREESOUND_TOKEN (Settings → Secrets).")
+        raise RuntimeError("Token FreeSound manquant. Ajoute le secret FREESOUND_TOKEN dans le Space.")
     c = freesound.FreesoundClient()
     c.set_token(FREESOUND_TOKEN, "token")
     return c
 
 
 # ============================================================
-# PARTIE A — OpenSMILE (upload)
+# DIAGNOSTIC FILE LISTS
+# ============================================================
+FILES_A = [
+    "xgb_model_EffectSound.pkl",
+    "xgb_model_Music.pkl",
+]
+
+FILES_B = [
+    "xgb_num_downloads_effectsound_model.pkl",
+    "xgb_num_downloads_effectsound_features.pkl",
+    "xgb_avg_rating_effectsound_model.pkl",
+    "xgb_avg_rating_effectsound_features.pkl",
+    "xgb_avg_rating_effectsound_label_encoder.pkl",
+    "xgb_num_downloads_music_model.pkl",
+    "xgb_num_downloads_music_features.pkl",
+    "xgb_avg_rating_music_model.pkl",
+    "xgb_avg_rating_music_features.pkl",
+    "xgb_avg_rating_music_label_encoder.pkl",
+]
+
+FILES_C_ROOT = [
+    "effectSound_model_num_downloads.joblib",
+    "effectSound_xgb_avg_rating.joblib",
+    "effectSound_xgb_avg_rating_label_encoder.joblib",
+    "effect_model_features_list.joblib",
+    "music_model_num_downloads.joblib",
+    "music_xgb_avg_rating.joblib",
+    "music_xgb_avg_rating_label_encoder.joblib",
+    # feature list music: tu as les deux, on accepte l’un ou l’autre
+    # "music_model_features_list.joblib" OU "model_features_list.joblib"
+]
+
+FILES_C_EFFECT_DIR = [
+    "effectSound/scaler_effectSamplerate.joblib",
+    "effectSound/scaler_effectSound_age_days_log.joblib",
+    "effectSound/username_freq_dict_effectSound.joblib",
+    "effectSound/est_num_downloads_effectSound.joblib",
+    "effectSound/avg_rating_transformer_effectSound.joblib",
+    "effectSound/effectSound_subcategory_cols.joblib",
+    "effectSound/effectSound_onehot_cols.joblib",
+    "effectSound/effect_onehot_tags.joblib",
+]
+
+FILES_C_MUSIC_DIR = [
+    "music/scaler_music_samplerate.joblib",
+    "music/scaler_music_age_days_log.joblib",
+    "music/username_freq_dict_music.joblib",
+    "music/est_num_downloads_music.joblib",
+    "music/avg_rating_transformer_music.joblib",
+    "music/music_subcategory_cols.joblib",
+    "music/music_onehot_cols.joblib",
+    "music/music_onehot_tags.joblib",
+]
+
+
+# ============================================================
+# PARTIE A — OpenSMILE upload
 # ============================================================
+A_MODELS = {}
+
+def load_A_models():
+    A_MODELS["effect"] = load_local("xgb_model_EffectSound.pkl")
+    A_MODELS["music"] = load_local("xgb_model_Music.pkl")
+
 SMILE = opensmile.Smile(
     feature_set=opensmile.FeatureSet.eGeMAPSv02,
     feature_level=opensmile.FeatureLevel.Functionals,
@@ -191,16 +234,6 @@ SMILE = opensmile.Smile(
 RATING_DISPLAY_AUDIO = {0: "❌ Informations manquantes", 1: "⭐ Faible", 2: "⭐⭐ Moyen", 3: "⭐⭐⭐ Élevé"}
 DOWNLOADS_DISPLAY_AUDIO = {0: "⭐ Faible", 1: "⭐⭐ Moyen", 2: "⭐⭐⭐ Élevé"}
 
-MODEL_EFFECT_A, PATH_EFFECT_A = load_joblib_any([
-    "xgb_model_EffectSound.pkl",
-    "xgb_model_effectsound.pkl",
-    "xgb_model_effectSound.pkl",
-])
-MODEL_MUSIC_A, PATH_MUSIC_A = load_joblib_any([
-    "xgb_model_Music.pkl",
-    "xgb_model_music.pkl",
-])
-
 def get_duration_seconds(filepath):
     ext = os.path.splitext(filepath)[1].lower()
     if ext == ".mp3":
@@ -218,6 +251,7 @@ def to_wav_16k_mono(filepath):
                     return filepath
         except Exception:
             pass
+
     audio = AudioSegment.from_file(filepath)
     audio = audio.set_channels(1).set_frame_rate(SR_TARGET)
     tmp = tempfile.NamedTemporaryFile(suffix=".wav", delete=False)
@@ -232,26 +266,12 @@ def extract_opensmile_features(filepath):
     return feats
 
 def predict_upload_with_dmatrix(model, X_df: pd.DataFrame):
-    # sklearn wrapper or Booster
     booster = model.get_booster() if hasattr(model, "get_booster") else model
     dm = xgb.DMatrix(X_df.values, feature_names=list(X_df.columns))
-    p = booster.predict(dm)
-    p = np.asarray(p)
-    if p.ndim == 1:
-        # si ton modèle renvoie 2 outputs concat, ça ne marche pas;
-        # ton modèle A semble renvoyer 2 classes (avg, downloads) -> souvent shape (2,)
-        # on force (1, -1)
-        p = p.reshape(1, -1)
-    return p
+    p_ = booster.predict(dm)
+    return np.asarray(p_).reshape(1, -1)
 
 def predict_opensmile_upload(audio_file):
-    if MODEL_EFFECT_A is None or MODEL_MUSIC_A is None:
-        return html_error(
-            "Modèles OpenSMILE manquants",
-            "Il faut fournir les deux modèles OpenSMILE (effect & music) à la racine, ex: "
-            "<code>xgb_model_EffectSound.pkl</code> et <code>xgb_model_Music.pkl</code>."
-        )
-
     if audio_file is None:
         return html_error("Aucun fichier", "Veuillez importer un fichier audio (wav, mp3, flac…).")
 
@@ -261,45 +281,46 @@ def predict_opensmile_upload(audio_file):
         return html_error("Audio illisible", f"Détail : <code>{e}</code>")
 
     if duration < MIN_EFFECT:
-        return html_error("Audio trop court", f"Durée : <b>{duration:.2f}s</b> — attendu 0.5–3s ou 10–60s")
+        return html_error("Audio trop court", f"Durée : <b>{duration:.2f}s</b><br>Accepté: 0.5–3s ou 10–60s")
     if (MAX_EFFECT < duration < MIN_MUSIC) or duration > MAX_MUSIC:
-        return html_error("Audio hors plage", f"Durée : <b>{duration:.2f}s</b> — attendu 0.5–3s ou 10–60s")
+        return html_error("Audio hors plage", f"Durée : <b>{duration:.2f}s</b><br>Accepté: 0.5–3s ou 10–60s")
+
+    try:
+        if not A_MODELS:
+            load_A_models()
+    except Exception as e:
+        return html_error("Modèles OpenSMILE manquants", f"Détail : <code>{e}</code>")
 
     if duration <= MAX_EFFECT:
-        badge, model = "🔊 OpenSMILE (upload) — EffectSound", MODEL_EFFECT_A
+        badge = "🔊 OpenSMILE (upload) — EffectSound"
+        model = A_MODELS["effect"]
     else:
-        badge, model = "🎵 OpenSMILE (upload) — Music", MODEL_MUSIC_A
+        badge = "🎵 OpenSMILE (upload) — Music"
+        model = A_MODELS["music"]
 
     try:
         X = extract_opensmile_features(audio_file)
     except Exception as e:
         return html_error("Extraction openSMILE échouée", f"Détail : <code>{e}</code>")
 
-    # align features si possible
     try:
-        if hasattr(model, "feature_names_in_"):
-            expected = list(model.feature_names_in_)
-            X = X.reindex(columns=expected, fill_value=0)
-    except Exception:
-        # pas bloquant
-        pass
+        expected = model.feature_names_in_ if hasattr(model, "feature_names_in_") else list(X.columns)
+        X = X.reindex(columns=list(expected), fill_value=0)
+    except Exception as e:
+        return html_error("Alignement features échoué", f"Détail : <code>{e}</code>")
 
     try:
         y = predict_upload_with_dmatrix(model, X)
     except Exception as e:
         return html_error("Prédiction échouée", f"Détail : <code>{e}</code>")
 
-    # Convention attendue : y[0,0]=avg_class, y[0,1]=dl_class
-    try:
-        avg_class = int(y[0, 0])
-        dl_class = int(y[0, 1])
-    except Exception:
-        return html_error("Sortie modèle inattendue", f"Forme sortie: <code>{getattr(y,'shape',None)}</code>")
+    avg_class = int(y[0, 0])
+    dl_class = int(y[0, 1])
 
     rating_text = RATING_DISPLAY_AUDIO.get(avg_class, "Inconnu")
     downloads_text = DOWNLOADS_DISPLAY_AUDIO.get(dl_class, "Inconnu")
+
     extra = f"""
-<div class="hint">Modèles chargés: <code>{PATH_EFFECT_A}</code> · <code>{PATH_MUSIC_A}</code></div>
 <div style="margin-top:12px; padding-top:10px; border-top:1px dashed #d1d5db">
 {interpret_results(avg_class, dl_class)}
 </div>
@@ -308,94 +329,27 @@ def predict_opensmile_upload(audio_file):
 
 
 # ============================================================
-# PARTIE B — FreeSound Acoustic Features (API fields)
-#   => c’est ici que tu as l’erreur de fichier manquant
+# PARTIE B — FreeSound API acoustic features
 # ============================================================
-def load_feature_models_B():
-    """
-    Essaie de trouver les fichiers même si tu as des variantes de nom.
-    Retourne dict + liste problèmes.
-    """
-    problems = []
-    M = {}
-
-    # MUSIC
-    M["music_num_model"], M["music_num_model_path"] = load_joblib_any([
-        "xgb_num_downloads_music_model.pkl",
-        "*num*downloads*music*model*.pkl",
-        "*num*downloads*music*model*.joblib",
-    ])
-    M["music_num_feats"], M["music_num_feats_path"] = load_joblib_any([
-        "xgb_num_downloads_music_features.pkl",
-        "*num*downloads*music*features*.pkl",
-        "*num*downloads*music*features*.joblib",
-    ])
-    M["music_avg_model"], M["music_avg_model_path"] = load_joblib_any([
-        "xgb_avg_rating_music_model.pkl",
-        "*avg*rating*music*model*.pkl",
-        "*avg*rating*music*model*.joblib",
-    ])
-    M["music_avg_feats"], M["music_avg_feats_path"] = load_joblib_any([
-        "xgb_avg_rating_music_features.pkl",
-        "*avg*rating*music*features*.pkl",
-        "*avg*rating*music*features*.joblib",
-    ])
-    M["music_avg_le"], M["music_avg_le_path"] = load_joblib_any([
-        "xgb_avg_rating_music_label_encoder.pkl",
-        "*avg*rating*music*label*encoder*.pkl",
-        "*avg*rating*music*label*encoder*.joblib",
-    ])
-
-    # EFFECTSOUND (variantes de nom)
-    M["eff_num_model"], M["eff_num_model_path"] = load_joblib_any([
-        "xgb_num_downloads_effectsound_model.pkl",
-        "xgb_num_downloads_effectSound_model.pkl",
-        "xgb_num_downloads_effect_sound_model.pkl",
-        "*num*downloads*effect*model*.pkl",
-        "*num*downloads*effect*model*.joblib",
-    ])
-    M["eff_num_feats"], M["eff_num_feats_path"] = load_joblib_any([
-        "xgb_num_downloads_effectsound_features.pkl",
-        "xgb_num_downloads_effectSound_features.pkl",
-        "xgb_num_downloads_effect_sound_features.pkl",
-        "*num*downloads*effect*features*.pkl",
-        "*num*downloads*effect*features*.joblib",
-    ])
-    M["eff_avg_model"], M["eff_avg_model_path"] = load_joblib_any([
-        "xgb_avg_rating_effectsound_model.pkl",
-        "xgb_avg_rating_effectSound_model.pkl",
-        "xgb_avg_rating_effect_sound_model.pkl",
-        "*avg*rating*effect*model*.pkl",
-        "*avg*rating*effect*model*.joblib",
-    ])
-    M["eff_avg_feats"], M["eff_avg_feats_path"] = load_joblib_any([
-        # <-- c’est exactement celui qui manque chez toi, on met plein de variantes
-        "xgb_avg_rating_effectsound_features.pkl",
-        "xgb_avg_rating_effectSound_features.pkl",
-        "xgb_avg_rating_effect_sound_features.pkl",
-        "*avg*rating*effect*features*.pkl",
-        "*avg*rating*effect*features*.joblib",
-    ])
-    M["eff_avg_le"], M["eff_avg_le_path"] = load_joblib_any([
-        "xgb_avg_rating_effectsound_label_encoder.pkl",
-        "xgb_avg_rating_effectSound_label_encoder.pkl",
-        "xgb_avg_rating_effect_sound_label_encoder.pkl",
-        "*avg*rating*effect*label*encoder*.pkl",
-        "*avg*rating*effect*label*encoder*.joblib",
-    ])
-
-    required = [
-        ("music_num_model", "music_num_feats", "music_avg_model", "music_avg_feats", "music_avg_le"),
-        ("eff_num_model", "eff_num_feats", "eff_avg_model", "eff_avg_feats", "eff_avg_le"),
-    ]
-    for group in required:
-        for k in group:
-            if M.get(k) is None:
-                problems.append(k)
-
-    return M, problems
-
-MODELS_B, PROBLEMS_B = load_feature_models_B()
+B_MODELS = {}
+
+def load_B_models():
+    # downloads
+    B_MODELS["eff_num_model"] = load_local("xgb_num_downloads_effectsound_model.pkl")
+    B_MODELS["eff_num_feats"] = load_local("xgb_num_downloads_effectsound_features.pkl")
+
+    B_MODELS["mus_num_model"] = load_local("xgb_num_downloads_music_model.pkl")
+    B_MODELS["mus_num_feats"] = load_local("xgb_num_downloads_music_features.pkl")
+
+    # avg rating
+    B_MODELS["eff_avg_model"] = load_local("xgb_avg_rating_effectsound_model.pkl")
+    B_MODELS["eff_avg_feats"] = load_local("xgb_avg_rating_effectsound_features.pkl")
+    B_MODELS["eff_avg_le"] = load_local("xgb_avg_rating_effectsound_label_encoder.pkl")
+
+    B_MODELS["mus_avg_model"] = load_local("xgb_avg_rating_music_model.pkl")
+    B_MODELS["mus_avg_feats"] = load_local("xgb_avg_rating_music_features.pkl")
+    B_MODELS["mus_avg_le"] = load_local("xgb_avg_rating_music_label_encoder.pkl")
+
 NUM_DOWNLOADS_MAP_B = {0: "Faible", 1: "Moyen", 2: "Élevé"}
 
 def predict_with_model_fs(model, features_dict, feat_list, label_encoder=None):
@@ -407,7 +361,7 @@ def predict_with_model_fs(model, features_dict, feat_list, label_encoder=None):
         row.append(safe_float(val))
 
     X = pd.DataFrame([row], columns=feat_list)
-    dmatrix = xgb.DMatrix(X.values, feature_names=list(feat_list))
+    dmatrix = xgb.DMatrix(X.values, feature_names=feat_list)
 
     booster = model.get_booster() if hasattr(model, "get_booster") else model
     pred_int = int(booster.predict(dmatrix)[0])
@@ -417,15 +371,6 @@ def predict_with_model_fs(model, features_dict, feat_list, label_encoder=None):
     return pred_int
 
 def predict_freesound_acoustic_features(url: str):
-    if PROBLEMS_B:
-        missing = ", ".join(f"<code>{k}</code>" for k in PROBLEMS_B)
-        files = "<br>".join(list_local_files()[:200])
-        return html_error(
-            "Modèles Features API incomplets",
-            f"Il manque des fichiers nécessaires au pipeline B :<br>{missing}<br><br>"
-            f"<b>Fichiers détectés dans ton Space (aperçu)</b>:<br><pre>{files}</pre>"
-        )
-
     if not url or not url.strip():
         return html_error("URL vide", "Colle une URL du type <code>https://freesound.org/s/123456/</code>")
 
@@ -439,12 +384,19 @@ def predict_freesound_acoustic_features(url: str):
     except Exception as e:
         return html_error("Token FreeSound", str(e))
 
-    # champs à récupérer
-    all_features = list(set(
-        MODELS_B["music_num_feats"] + MODELS_B["music_avg_feats"] +
-        MODELS_B["eff_num_feats"] + MODELS_B["eff_avg_feats"]
-    ))
-    fields = "duration," + ",".join(all_features)
+    try:
+        if not B_MODELS:
+            load_B_models()
+    except Exception as e:
+        return html_error("Modèles Features API manquants", f"Détail : <code>{e}</code>")
+
+    # champs API = union de toutes les features nécessaires (pour éviter de faire 2 appels)
+    all_feats = set()
+    all_feats.update(B_MODELS["eff_num_feats"])
+    all_feats.update(B_MODELS["mus_num_feats"])
+    all_feats.update(B_MODELS["eff_avg_feats"])
+    all_feats.update(B_MODELS["mus_avg_feats"])
+    fields = "duration," + ",".join(sorted(all_feats))
 
     try:
         results = fs_client.search(query="", filter=f"id:{sound_id}", fields=fields)
@@ -457,107 +409,430 @@ def predict_freesound_acoustic_features(url: str):
     sound = results.results[0]
     duration = safe_float(sound.get("duration", 0))
 
-    if MIN_EFFECT <= duration <= MAX_EFFECT:
-        badge = "🔊 FreeSound (Features acoustiques API) — EffectSound"
-        dl_class = int(predict_with_model_fs(MODELS_B["eff_num_model"], sound, MODELS_B["eff_num_feats"]))
-        avg_text = str(predict_with_model_fs(MODELS_B["eff_avg_model"], sound, MODELS_B["eff_avg_feats"], MODELS_B["eff_avg_le"]))
+    if duration < MIN_EFFECT or ((MAX_EFFECT < duration < MIN_MUSIC) or duration > MAX_MUSIC):
+        return html_error("Durée non supportée", f"Durée : <b>{duration:.2f}s</b><br>Accepté: 0.5–3s ou 10–60s")
+
+    # EffectSound
+    if duration <= MAX_EFFECT:
+        badge = "🔊 FreeSound (API features acoustiques) — EffectSound"
+        dl_class = int(predict_with_model_fs(B_MODELS["eff_num_model"], sound, B_MODELS["eff_num_feats"]))
         dl_text = NUM_DOWNLOADS_MAP_B.get(dl_class, str(dl_class))
+        avg_text = str(predict_with_model_fs(B_MODELS["eff_avg_model"], sound, B_MODELS["eff_avg_feats"], B_MODELS["eff_avg_le"]))
         avg_class = avg_label_to_class(avg_text)
 
         extra = f"""
-<div class="hint">ID: <b>{sound_id}</b></div>
+<div class="hint">ID FreeSound : <b>{sound_id}</b></div>
 <div style="margin-top:12px; padding-top:10px; border-top:1px dashed #d1d5db">
 {interpret_results(avg_class, dl_class)}
 </div>
 """
         return html_result(badge, duration, avg_text, dl_text, extra_html=extra)
 
-    if MIN_MUSIC <= duration <= MAX_MUSIC:
-        badge = "🎵 FreeSound (Features acoustiques API) — Music"
-        dl_class = int(predict_with_model_fs(MODELS_B["music_num_model"], sound, MODELS_B["music_num_feats"]))
-        avg_text = str(predict_with_model_fs(MODELS_B["music_avg_model"], sound, MODELS_B["music_avg_feats"], MODELS_B["music_avg_le"]))
-        dl_text = NUM_DOWNLOADS_MAP_B.get(dl_class, str(dl_class))
-        avg_class = avg_label_to_class(avg_text)
+    # Music
+    badge = "🎵 FreeSound (API features acoustiques) — Music"
+    dl_class = int(predict_with_model_fs(B_MODELS["mus_num_model"], sound, B_MODELS["mus_num_feats"]))
+    dl_text = NUM_DOWNLOADS_MAP_B.get(dl_class, str(dl_class))
+    avg_text = str(predict_with_model_fs(B_MODELS["mus_avg_model"], sound, B_MODELS["mus_avg_feats"], B_MODELS["mus_avg_le"]))
+    avg_class = avg_label_to_class(avg_text)
 
-        extra = f"""
-<div class="hint">ID: <b>{sound_id}</b></div>
+    extra = f"""
+<div class="hint">ID FreeSound : <b>{sound_id}</b></div>
 <div style="margin-top:12px; padding-top:10px; border-top:1px dashed #d1d5db">
 {interpret_results(avg_class, dl_class)}
 </div>
 """
-        return html_result(badge, duration, avg_text, dl_text, extra_html=extra)
-
-    return html_error("Durée non supportée", f"Durée : <b>{duration:.2f}s</b> — attendu 0.5–3s ou 10–60s")
+    return html_result(badge, duration, avg_text, dl_text, extra_html=extra)
 
 
 # ============================================================
-# PARTIE C — Metadata (désactivée si pas de dossiers/fichiers)
+# PARTIE C — Metadata preprocessing + joblib
 # ============================================================
-def predict_freesound_metadata_stub(url: str):
-    return html_error(
-        "Pipeline Metadata non disponible",
-        "Tu as dit ne pas avoir les dossiers <code>music/</code> et <code>effectSound/</code> "
-        "et/ou les joblib de preprocessing. Donc je n’active pas ce pipeline pour éviter de crasher."
-        "<br><br>Si tu veux l’activer : ajoute les joblib de preprocessing + les modèles metadata, "
-        "ou dis-moi comment tu les as nommés/organisés."
+C_READY = False
+C = {}
+C_LOAD_ERRORS = []
+
+def try_load_C():
+    global C_READY, C, C_LOAD_ERRORS
+    C_READY = False
+    C = {}
+    C_LOAD_ERRORS = []
+
+    def load_and_store(key, relpath):
+        try:
+            C[key] = load_local(relpath)
+            return True
+        except Exception as e:
+            C_LOAD_ERRORS.append(f"{relpath} -> {type(e).__name__}: {e}")
+            return False
+
+    ok = True
+
+    # preprocess music
+    ok &= load_and_store("scaler_samplerate_music",   "music/scaler_music_samplerate.joblib")
+    ok &= load_and_store("scaler_age_days_music",     "music/scaler_music_age_days_log.joblib")
+    ok &= load_and_store("username_freq_music",       "music/username_freq_dict_music.joblib")
+    ok &= load_and_store("est_num_downloads_music",   "music/est_num_downloads_music.joblib")
+    ok &= load_and_store("avg_rating_tr_music",       "music/avg_rating_transformer_music.joblib")
+    ok &= load_and_store("music_subcat_cols",         "music/music_subcategory_cols.joblib")
+    ok &= load_and_store("music_onehot_cols",         "music/music_onehot_cols.joblib")
+    ok &= load_and_store("music_onehot_tags",         "music/music_onehot_tags.joblib")
+
+    # preprocess effect
+    ok &= load_and_store("scaler_samplerate_effect",  "effectSound/scaler_effectSamplerate.joblib")
+    ok &= load_and_store("scaler_age_days_effect",    "effectSound/scaler_effectSound_age_days_log.joblib")
+    ok &= load_and_store("username_freq_effect",      "effectSound/username_freq_dict_effectSound.joblib")
+    ok &= load_and_store("est_num_downloads_effect",  "effectSound/est_num_downloads_effectSound.joblib")
+    ok &= load_and_store("avg_rating_tr_effect",      "effectSound/avg_rating_transformer_effectSound.joblib")
+    ok &= load_and_store("effect_subcat_cols",        "effectSound/effectSound_subcategory_cols.joblib")
+    ok &= load_and_store("effect_onehot_cols",        "effectSound/effectSound_onehot_cols.joblib")
+    ok &= load_and_store("effect_onehot_tags",        "effectSound/effect_onehot_tags.joblib")
+
+    # models root
+    ok &= load_and_store("music_nd_model",            "music_model_num_downloads.joblib")
+    ok &= load_and_store("music_ar_model",            "music_xgb_avg_rating.joblib")
+    ok &= load_and_store("music_ar_le",               "music_xgb_avg_rating_label_encoder.joblib")
+
+    ok &= load_and_store("effect_nd_model",           "effectSound_model_num_downloads.joblib")
+    ok &= load_and_store("effect_ar_model",           "effectSound_xgb_avg_rating.joblib")
+    ok &= load_and_store("effect_ar_le",              "effectSound_xgb_avg_rating_label_encoder.joblib")
+
+    # feature lists
+    if exists("music_model_features_list.joblib"):
+        ok &= load_and_store("music_features", "music_model_features_list.joblib")
+    elif exists("model_features_list.joblib"):
+        ok &= load_and_store("music_features", "model_features_list.joblib")
+    else:
+        ok = False
+        C_LOAD_ERRORS.append("music features list manquante: music_model_features_list.joblib OU model_features_list.joblib")
+
+    ok &= load_and_store("effect_features",           "effect_model_features_list.joblib")
+
+    if ok:
+        C["music_features"] = list(dict.fromkeys(C["music_features"]))
+        C["effect_features"] = list(dict.fromkeys(C["effect_features"]))
+        C_READY = True
+    else:
+        C_READY = False
+
+# run once at import
+try_load_C()
+
+def preprocess_name(df, vec_dim=8):
+    df = df.copy()
+    df["name_len"] = df["name_clean"].str.len()
+    vectorizer = HashingVectorizer(n_features=vec_dim, alternate_sign=False, norm=None)
+    name_vec_sparse = vectorizer.transform(df["name_clean"])
+    name_vec_df = pd.DataFrame(
+        name_vec_sparse.toarray(),
+        columns=[f"name_vec_{i}" for i in range(vec_dim)],
+        index=df.index
     )
+    return pd.concat([df, name_vec_df], axis=1)
+
+def fetch_sound_metadata(fs_client, sound_url):
+    sound_id = parse_sound_id(sound_url)
+    sound = fs_client.get_sound(sound_id)
+    data = {
+        "id": sound_id,
+        "name": sound.name,
+        "num_ratings": getattr(sound, "num_ratings", 0),
+        "tags": ",".join(sound.tags) if getattr(sound, "tags", None) else "",
+        "username": getattr(sound, "username", ""),
+        "description": getattr(sound, "description", "") or "",
+        "created": getattr(sound, "created", ""),
+        "license": getattr(sound, "license", ""),
+        "num_downloads": getattr(sound, "num_downloads", 0),
+        "channels": getattr(sound, "channels", 0),
+        "filesize": getattr(sound, "filesize", 0),
+        "num_comments": getattr(sound, "num_comments", 0),
+        "category_is_user_provided": getattr(sound, "category_is_user_provided", 0),
+        "duration": getattr(sound, "duration", 0),
+        "avg_rating": getattr(sound, "avg_rating", 0),
+        "category": getattr(sound, "category", "Unknown"),
+        "subcategory": getattr(sound, "subcategory", "Other"),
+        "type": getattr(sound, "type", ""),
+        "samplerate": getattr(sound, "samplerate", 0),
+    }
+    return pd.DataFrame([data])
+
+def preprocess_sound_metadata(df):
+    df = df.copy()
+    dur = float(df["duration"].iloc[0])
+
+    if MIN_EFFECT <= dur <= MAX_EFFECT:
+        dataset_type = "effectSound"
+        scaler_samplerate = C["scaler_samplerate_effect"]
+        scaler_age = C["scaler_age_days_effect"]
+        username_freq = C["username_freq_effect"]
+        est_num_downloads = C["est_num_downloads_effect"]
+        avg_rating_tr = C["avg_rating_tr_effect"]
+        subcat_cols = C["effect_subcat_cols"]
+        onehot_cols = C["effect_onehot_cols"]
+        onehot_tags = C["effect_onehot_tags"]
+    elif MIN_MUSIC <= dur <= MAX_MUSIC:
+        dataset_type = "music"
+        scaler_samplerate = C["scaler_samplerate_music"]
+        scaler_age = C["scaler_age_days_music"]
+        username_freq = C["username_freq_music"]
+        est_num_downloads = C["est_num_downloads_music"]
+        avg_rating_tr = C["avg_rating_tr_music"]
+        subcat_cols = C["music_subcat_cols"]
+        onehot_cols = C["music_onehot_cols"]
+        onehot_tags = C["music_onehot_tags"]
+    else:
+        return None, None, f"Durée hors plage ({dur:.2f}s)."
+
+    df["category_is_user_provided"] = df["category_is_user_provided"].astype(int)
+    df["username_freq"] = df["username"].map(username_freq).fillna(0)
+
+    for col in ["num_ratings", "num_comments", "filesize", "duration"]:
+        df[col] = np.log1p(df[col])
+
+    df["samplerate"] = scaler_samplerate.transform(df[["samplerate"]])
+
+    df["created"] = pd.to_datetime(df["created"], errors="coerce").dt.tz_localize(None)
+    df["age_days"] = (pd.Timestamp.now() - df["created"]).dt.days
+    df["age_days_log"] = np.log1p(df["age_days"])
+    df["age_days_log_scaled"] = scaler_age.transform(df[["age_days_log"]])
+    df = df.drop(columns=["created", "age_days", "age_days_log"], errors="ignore")
+
+    df["num_downloads_class"] = est_num_downloads.transform(df[["num_downloads"]])
+    df["avg_rating"] = avg_rating_tr.transform(df["avg_rating"].to_numpy())
+
+    for col in subcat_cols:
+        df[col] = 0
+    subcat_val = df["subcategory"].iloc[0]
+    for col in subcat_cols:
+        cat_name = col.replace("subcategory_", "")
+        if subcat_val == cat_name:
+            df[col] = 1
+    df.drop(columns=["subcategory"], inplace=True, errors="ignore")
+
+    for col in onehot_cols:
+        if col not in df.columns:
+            df[col] = 0
+
+    license_val = df.loc[0, "license"]
+    category_val = df.loc[0, "category"]
+    type_val = df.loc[0, "type"]
+
+    for col_name in [f"license_{license_val}", f"category_{category_val}", f"type_{type_val}"]:
+        if col_name in df.columns:
+            df[col_name] = 1
+
+    for col in onehot_tags:
+        if col not in df.columns:
+            df[col] = 0
+
+    tags_list = df["tags"].iloc[0].lower().split(",") if df["tags"].iloc[0] else []
+    for col in onehot_tags:
+        tag_name = col.replace("tag_", "").lower()
+        if tag_name in tags_list:
+            df[col] = 1
+    df.drop(columns=["tags"], inplace=True, errors="ignore")
+
+    df["name_clean"] = df["name"].astype(str).str.lower().str.rsplit(".", n=1).str[0]
+    df = preprocess_name(df, vec_dim=8)
+    df.drop(columns=["name", "name_clean"], inplace=True, errors="ignore")
+
+    # IMPORTANT: Pas de GloVe ici (ton modèle est piloté par model_features_list.joblib)
+    df.drop(columns=["description"], inplace=True, errors="ignore")
+
+    df.drop(columns=["license", "category", "type", "id", "num_downloads", "username"], inplace=True, errors="ignore")
+
+    return df, dataset_type, None
+
+def predict_with_model_df(model, df_input):
+    booster_feats = model.get_booster().feature_names
+    X_aligned = df_input.reindex(columns=booster_feats, fill_value=0.0).astype(float)
+    dmatrix = xgb.DMatrix(X_aligned.values, feature_names=booster_feats)
+    pred = model.get_booster().predict(dmatrix)
+    pred_val = pred[0]
+    if hasattr(pred_val, "__len__") and np.size(pred_val) > 1:
+        return int(np.argmax(pred_val))
+    return int(round(float(pred_val)))
+
+def predict_freesound_metadata(url: str, show_debug: bool):
+    if not C_READY:
+        body = "Le pipeline metadata n’a pas pu charger tous les joblib."
+        if C_LOAD_ERRORS:
+            body += "<br><br><details><summary><b>Voir erreurs</b></summary><pre>" + "\n".join(C_LOAD_ERRORS[:80]) + "</pre></details>"
+        return html_warn("Pipeline C désactivé", body)
+
+    if not url or not url.strip():
+        return html_error("URL vide", "Colle une URL du type <code>https://freesound.org/s/123456/</code>")
+
+    try:
+        sound_id = parse_sound_id(url)
+    except Exception:
+        return html_error("URL invalide", "Impossible d'extraire l'ID depuis l'URL.")
+
+    try:
+        fs_client = get_fs_client()
+    except Exception as e:
+        return html_error("Token FreeSound", str(e))
+
+    try:
+        df_raw = fetch_sound_metadata(fs_client, url)
+    except Exception as e:
+        return html_error("Erreur API FreeSound", f"Détail : <code>{e}</code>")
+
+    dur = float(df_raw["duration"].iloc[0])
+    if dur < MIN_EFFECT or ((MAX_EFFECT < dur < MIN_MUSIC) or dur > MAX_MUSIC):
+        return html_error("Durée non supportée", f"Durée : <b>{dur:.2f}s</b><br>Accepté: 0.5–3s ou 10–60s")
+
+    df_proc, dtype, err = preprocess_sound_metadata(df_raw)
+    if df_proc is None:
+        return html_error("Prétraitement metadata", err or "Erreur inconnue.")
+
+    if dtype == "effectSound":
+        badge = "🔊 FreeSound (metadata) — EffectSound"
+        nd_model = C["effect_nd_model"]
+        ar_model = C["effect_ar_model"]
+        ar_le = C["effect_ar_le"]
+        feats = C["effect_features"]
+    else:
+        badge = "🎵 FreeSound (metadata) — Music"
+        nd_model = C["music_nd_model"]
+        ar_model = C["music_ar_model"]
+        ar_le = C["music_ar_le"]
+        feats = C["music_features"]
+
+    df_for_model = df_proc.reindex(columns=feats, fill_value=0.0).astype(float)
+
+    dl_class = predict_with_model_df(nd_model, df_for_model)
+    dl_map = {0: "Low", 1: "Medium", 2: "High"}
+    dl_text = dl_map.get(dl_class, str(dl_class))
+
+    ar_class = predict_with_model_df(ar_model, df_for_model)
+    try:
+        avg_text = ar_le.inverse_transform([ar_class])[0]
+    except Exception:
+        avg_text = f"Classe {ar_class}"
+
+    avg_class_for_interp = avg_label_to_class(avg_text)
+    dl_class_for_interp = {"Low": 0, "Medium": 1, "High": 2}.get(dl_text, 1)
+
+    debug_html = ""
+    if show_debug:
+        raw_txt = "\n".join([f"{c}: {df_raw.loc[0,c]}" for c in df_raw.columns])
+        proc_cols = df_proc.columns.tolist()
+        proc_preview = proc_cols[:140]
+        proc_txt = "\n".join([f"{c}: {df_proc.loc[0,c]}" for c in proc_preview])
+        debug_html = f"""
+<div style="margin-top:12px; padding-top:10px; border-top:1px dashed #d1d5db">
+  <details><summary><b>Debug</b> — métadonnées brutes</summary><pre>{raw_txt}</pre></details>
+  <details><summary><b>Debug</b> — features après preprocessing (aperçu)</summary><pre>{proc_txt}</pre></details>
+</div>
+"""
+
+    extra = f"""
+<div class="hint">ID FreeSound : <b>{sound_id}</b></div>
+<div style="margin-top:12px; padding-top:10px; border-top:1px dashed #d1d5db">
+{interpret_results(avg_class_for_interp, dl_class_for_interp)}
+</div>
+{debug_html}
+"""
+    return html_result(badge, dur, str(avg_text), str(dl_text), extra_html=extra)
 
 
 # ============================================================
-# Page “diagnostic”
+# DIAGNOSTIC HTML
 # ============================================================
-def status_page():
-    files = list_local_files()
-    files_txt = "\n".join(files) if files else "(aucun fichier .pkl/.joblib détecté)"
-    a_ok = (MODEL_EFFECT_A is not None and MODEL_MUSIC_A is not None)
-    b_ok = (len(PROBLEMS_B) == 0)
+def make_diagnostic_html():
+    # A
+    missing_a = [f for f in FILES_A if not exists(f)]
+    a_ok = (len(missing_a) == 0)
+
+    # B
+    missing_b = [f for f in FILES_B if not exists(f)]
+    b_ok = (len(missing_b) == 0)
+
+    # C presence (files) + runtime load status (C_READY)
+    missing_c = []
+    for f in FILES_C_ROOT + FILES_C_EFFECT_DIR + FILES_C_MUSIC_DIR:
+        if not exists(f):
+            missing_c.append(f)
+    # music features list special rule
+    if not (exists("music_model_features_list.joblib") or exists("model_features_list.joblib")):
+        missing_c.append("music_model_features_list.joblib OU model_features_list.joblib")
+    c_files_ok = (len(missing_c) == 0)
+
+    parts = []
+    parts.append("<b>📦 Diagnostic du Space</b><br><br>")
+
+    parts.append("<b>OpenSMILE (A)</b><br>")
+    if a_ok:
+        parts.append("✅ OK<br>")
+        parts.append("Effect: xgb_model_EffectSound.pkl<br>Music: xgb_model_Music.pkl<br><br>")
+    else:
+        parts.append("❌ incomplet<br>")
+        parts.append(f"Manquants: {', '.join(missing_a)}<br><br>")
 
-    return f"""
-<div class="card">
-  <div class="card-title">📦 Diagnostic du Space</div>
-  <div class="grid">
-    <div class="box">
-      <div class="box-title">OpenSMILE (A)</div>
-      <div class="box-value">{'✅ OK' if a_ok else '❌ modèles manquants'}</div>
-      <div class="hint">Effect: <code>{PATH_EFFECT_A or 'non chargé'}</code><br>Music: <code>{PATH_MUSIC_A or 'non chargé'}</code></div>
-    </div>
-    <div class="box">
-      <div class="box-title">Features API (B)</div>
-      <div class="box-value">{'✅ OK' if b_ok else '❌ incomplet'}</div>
-      <div class="hint">Manquants: <code>{', '.join(PROBLEMS_B) if PROBLEMS_B else 'aucun'}</code></div>
-    </div>
-    <div class="box">
-      <div class="box-title">Metadata (C)</div>
-      <div class="box-value">⚠️ désactivé si dossiers/joblib absents</div>
-      <div class="hint">Activer seulement si preprocessing joblib présents.</div>
-    </div>
-  </div>
-  <div class="hint" style="margin-top:12px"><b>Fichiers détectés</b> :</div>
-  <pre>{files_txt}</pre>
-</div>
-""".strip()
+    parts.append("<b>Features API (B)</b><br>")
+    if b_ok:
+        parts.append("✅ OK<br><br>")
+    else:
+        parts.append("❌ incomplet<br>")
+        parts.append(f"Manquants: {', '.join(missing_b)}<br><br>")
+
+    parts.append("<b>Metadata (C)</b><br>")
+    if not c_files_ok:
+        parts.append("⚠️ désactivé si dossiers/joblib absents<br>")
+        parts.append("Activer seulement si preprocessing joblib présents.<br>")
+        parts.append(f"Manquants: {', '.join(missing_c)}<br><br>")
+    else:
+        # files are OK, but loading can still fail due to version mismatch
+        if C_READY:
+            parts.append("✅ OK (actif)<br><br>")
+        else:
+            parts.append("⚠️ fichiers présents mais chargement joblib a échoué (versions ?) <br>")
+            if C_LOAD_ERRORS:
+                parts.append("<details><summary><b>Voir erreurs de chargement</b></summary>")
+                parts.append("<pre>" + "\n".join(C_LOAD_ERRORS[:80]) + "</pre></details>")
+            parts.append("<br>")
+
+    # list detected files
+    detected = []
+    for root, _, files in os.walk(BASE_DIR):
+        for fn in files:
+            rel = os.path.relpath(os.path.join(root, fn), BASE_DIR)
+            detected.append(rel)
+    detected = sorted(detected)
+
+    parts.append("<details><summary><b>Fichiers détectés</b></summary>")
+    parts.append("<pre>" + "\n".join(detected) + "</pre></details>")
+
+    return html_box("Diagnostic", "".join(parts))
+
+
+def refresh_diagnostic():
+    # reload C on refresh
+    try_load_C()
+    return make_diagnostic_html()
 
 
 # ============================================================
-# GRADIO UI
+# GRADIO APP
 # ============================================================
-with gr.Blocks(title="Popularité FreeSound — Pipelines séparés", css=CSS, theme=gr.themes.Soft()) as demo:
+diag_init = make_diagnostic_html()
+
+with gr.Blocks(title="Popularité FreeSound — 3 pipelines", css=CSS, theme=gr.themes.Soft()) as demo:
     gr.HTML(f"""
-<div id="header-title">Popularité FreeSound — Pipelines séparés</div>
+<div id="header-title">Popularité FreeSound — 3 pipelines</div>
 <p id="header-sub">
-<b>A)</b> Upload → OpenSMILE<br>
-<b>B)</b> URL → Features acoustiques FreeSound (API fields)<br>
-<b>C)</b> URL → Metadata FreeSound (désactivé si fichiers absents)<br><br>
+<b>A)</b> Upload audio → <b>OpenSMILE</b><br>
+<b>B)</b> URL FreeSound → <b>Features acoustiques via API fields</b><br>
+<b>C)</b> URL FreeSound → <b>Metadata + preprocessing (joblib)</b><br><br>
 <b>Durées acceptées :</b> 🔊 {MIN_EFFECT}–{MAX_EFFECT}s · 🎵 {MIN_MUSIC}–{MAX_MUSIC}s
 </p>
 """)
 
-    with gr.Tabs():
-        with gr.Tab("📦 Diagnostic"):
-            diag = gr.HTML(value=status_page())
-            btn_refresh = gr.Button("Rafraîchir diagnostic")
-            btn_refresh.click(lambda: status_page(), outputs=diag)
+    diag_out = gr.HTML(value=diag_init)
+    btn_diag = gr.Button("🔄 Rafraîchir diagnostic")
+    btn_diag.click(refresh_diagnostic, outputs=diag_out)
 
+    with gr.Tabs():
         with gr.Tab("A) Upload → OpenSMILE"):
             with gr.Row():
                 with gr.Column():
@@ -580,9 +855,10 @@ with gr.Blocks(title="Popularité FreeSound — Pipelines séparés", css=CSS, t
             with gr.Row():
                 with gr.Column():
                     url_in = gr.Textbox(label="URL FreeSound", placeholder="https://freesound.org/s/123456/")
+                    show_debug = gr.Checkbox(label="Afficher debug (brut + aperçu features)", value=False)
                     btn = gr.Button("🚀 Prédire (Metadata)", variant="primary")
                 with gr.Column():
                     out = gr.HTML()
-            btn.click(predict_freesound_metadata_stub, inputs=url_in, outputs=out)
+            btn.click(predict_freesound_metadata, inputs=[url_in, show_debug], outputs=out)
 
 demo.launch()

requirements.txt

@@ -1,22 +1,15 @@
-gradio
-pandas
-numpy
-scikit-learn
-joblib
-xgboost
-soundfile
-pydub
-opensmile
-requests
-pytz
-gradio
-pandas
-numpy
-joblib
-xgboost
-requests
-urllib3
-scikit-learn
-imblearn
-matplotlib
+gradio==4.44.1
+pandas==2.2.2
+numpy==1.26.4
+scikit-learn==1.3.2
+joblib==1.3.2
+xgboost==2.0.3
+soundfile==0.12.1
+pydub==0.25.1
+opensmile==2.5.0
+requests==2.32.3
+pytz==2024.1
+urllib3==2.2.2
+matplotlib==3.8.4
+imbalanced-learn==0.11.0
 git+https://github.com/MTG/freesound-python.git