upload

app.py

@@ -0,0 +1,585 @@
+import html
+from datetime import datetime, timezone
+
+import gradio as gr
+import numpy as np
+import pandas as pd
+from sentence_transformers import SentenceTransformer
+
+
+CSV_PATH = "hf_atlas_ai_ml_interests_embedded.csv"
+EMBEDDINGS_PATH = "embeddings_ai_ml_interests.npy"
+MODEL_NAME = "BAAI/bge-small-en-v1.5"
+
+
+profile_df = pd.read_csv(CSV_PATH)
+profile_embeddings = np.load(EMBEDDINGS_PATH).astype(np.float32)
+
+print(f"✅ Loaded {len(profile_df)} HF Atlas profiles")
+print(f"✅ Loaded embeddings: {profile_embeddings.shape}")
+
+if len(profile_df) != profile_embeddings.shape[0]:
+    raise ValueError(
+        f"CSV / embeddings mismatch: {len(profile_df)} rows vs {profile_embeddings.shape[0]} embeddings"
+    )
+
+
+def detect_username_col(df):
+    for col in ["user", "username", "namespace"]:
+        if col in df.columns:
+            return col
+    return None
+
+
+USERNAME_COL = detect_username_col(profile_df)
+
+if USERNAME_COL is None:
+    raise ValueError("No username column found. Expected one of: user, username, namespace")
+
+
+def normalize_embeddings_if_needed(embeddings):
+    norms = np.linalg.norm(embeddings, axis=1, keepdims=True)
+    norms = np.where(norms == 0, 1.0, norms)
+    return embeddings / norms
+
+
+profile_embeddings = normalize_embeddings_if_needed(profile_embeddings)
+
+embedder = SentenceTransformer(MODEL_NAME)
+
+
+def safe_text(value, default=""):
+    if value is None:
+        return default
+
+    try:
+        if pd.isna(value):
+            return default
+    except Exception:
+        pass
+
+    text = str(value).strip()
+
+    if text.lower() in {"nan", "none", "null"}:
+        return default
+
+    return text
+
+
+def parse_last_seen(value):
+    text = safe_text(value)
+
+    if not text:
+        return pd.NaT
+
+    return pd.to_datetime(text, errors="coerce", utc=True)
+
+
+def prepare_dates():
+    if "last_seen_all_repo" not in profile_df.columns:
+        profile_df["_last_seen_dt"] = pd.NaT
+        print("⚠️ Column last_seen_all_repo not found. Date filter will only work as no-filter.")
+        return
+
+    profile_df["_last_seen_dt"] = profile_df["last_seen_all_repo"].map(parse_last_seen)
+
+    known = int(profile_df["_last_seen_dt"].notna().sum())
+    unknown = int(profile_df["_last_seen_dt"].isna().sum())
+
+    print(f"🕒 Known last_seen_all_repo dates: {known}")
+    print(f"🕳️ Unknown last_seen_all_repo dates: {unknown}")
+
+    if known > 0:
+        print(f"🕒 Min last_seen: {profile_df['_last_seen_dt'].min()}")
+        print(f"🕒 Max last_seen: {profile_df['_last_seen_dt'].max()}")
+
+
+prepare_dates()
+
+
+def filter_by_activity(df, activity_filter, custom_days):
+    if "_last_seen_dt" not in df.columns:
+        return df
+
+    custom_days = int(custom_days) if custom_days else 0
+
+    if activity_filter == "No filter":
+        return df
+
+    if activity_filter == "Has known activity date":
+        return df[df["_last_seen_dt"].notna()]
+
+    if activity_filter == "No known activity date":
+        return df[df["_last_seen_dt"].isna()]
+
+    if activity_filter == "Max age in days":
+        if custom_days <= 0:
+            return df
+
+        now = pd.Timestamp(datetime.now(timezone.utc))
+        cutoff = now - pd.Timedelta(days=custom_days)
+
+        return df[df["_last_seen_dt"].notna() & (df["_last_seen_dt"] >= cutoff)]
+
+    return df
+
+
+def format_number(value):
+    text = safe_text(value, "0")
+
+    try:
+        number = int(float(text))
+        return f"{number:,}".replace(",", " ")
+    except Exception:
+        return html.escape(text)
+
+
+def format_date(value):
+    text = safe_text(value)
+
+    if not text:
+        return "unknown"
+
+    try:
+        dt = pd.to_datetime(text, errors="coerce", utc=True)
+        if pd.isna(dt):
+            return "unknown"
+        return dt.strftime("%Y-%m-%d")
+    except Exception:
+        return html.escape(text)
+
+
+def truncate(text, max_len=900):
+    text = safe_text(text)
+
+    if len(text) <= max_len:
+        return text
+
+    return text[:max_len].rsplit(" ", 1)[0] + "..."
+
+
+def get_profile_url(row):
+    if "atlas_request_url" in row and safe_text(row["atlas_request_url"]):
+        return (
+            safe_text(row["atlas_request_url"])
+            .replace("/api/users/", "/")
+            .replace("/overview", "")
+        )
+
+    username = safe_text(row[USERNAME_COL])
+    return f"https://huggingface.co/{username}"
+
+
+def render_profile_card(row, score, rank):
+    username = safe_text(row[USERNAME_COL], "unknown")
+    fullname = safe_text(row.get("fullname", ""), "")
+    details = safe_text(row.get("details", ""), "")
+    ai_ml_interests = truncate(row.get("ai_ml_interests", ""), 800)
+    last_seen = format_date(row.get("last_seen_all_repo", ""))
+
+    num_models = format_number(row.get("numModels", row.get("n_models", 0)))
+    num_datasets = format_number(row.get("numDatasets", row.get("n_datasets", 0)))
+    num_spaces = format_number(row.get("numSpaces", row.get("n_spaces", 0)))
+    followers = format_number(row.get("numFollowers", 0))
+    likes = format_number(row.get("numLikes", row.get("numUpvotes", 0)))
+
+    url = get_profile_url(row)
+
+    title = html.escape(username)
+    fullname_html = html.escape(fullname) if fullname else "—"
+    details_html = html.escape(truncate(details, 300)) if details else ""
+    interests_html = html.escape(ai_ml_interests).replace("\n", "<br>")
+
+    extra_details = ""
+    if details_html:
+        extra_details = f"""
+        <div class="details">{details_html}</div>
+        """
+
+    return f"""
+    <div class="result-card">
+        <div class="result-topline">
+            <div>
+                <div class="rank">#{rank}</div>
+                <a class="username" href="{url}" target="_blank">{title}</a>
+                <div class="fullname">{fullname_html}</div>
+            </div>
+            <div class="score">{score * 100:.2f}%</div>
+        </div>
+
+        {extra_details}
+
+        <div class="interests">
+            <div class="label">AI/ML interests</div>
+            <div>{interests_html}</div>
+        </div>
+
+        <div class="stats">
+            <span>🧠 Models: <b>{num_models}</b></span>
+            <span>📚 Datasets: <b>{num_datasets}</b></span>
+            <span>🚀 Spaces: <b>{num_spaces}</b></span>
+            <span>❤️ Likes: <b>{likes}</b></span>
+            <span>👥 Followers: <b>{followers}</b></span>
+            <span>🕒 Last seen: <b>{last_seen}</b></span>
+        </div>
+    </div>
+    """
+
+
+def build_search_results(query, activity_filter, custom_days, display_count):
+    query = safe_text(query)
+
+    if not query:
+        return """
+        <div class="empty-state">
+            Describe an AI/ML topic, research area, tool, model family, or technical interest.
+        </div>
+        """, 0, False
+
+    custom_days = int(custom_days) if custom_days else 0
+    display_count = int(display_count)
+
+    eligible = filter_by_activity(profile_df, activity_filter, custom_days)
+
+    print("FILTER:", activity_filter, "DAYS:", custom_days, "ELIGIBLE:", len(eligible))
+
+    if "_last_seen_dt" in eligible.columns and len(eligible) > 0:
+        known_eligible = eligible[eligible["_last_seen_dt"].notna()]
+        if len(known_eligible) > 0:
+            print("ELIGIBLE MIN LAST_SEEN:", known_eligible["_last_seen_dt"].min())
+            print("ELIGIBLE MAX LAST_SEEN:", known_eligible["_last_seen_dt"].max())
+
+    if len(eligible) == 0:
+        return """
+        <div class="empty-state">
+            No profile found for this activity filter.
+        </div>
+        """, display_count, False
+
+    eligible_indices = eligible.index.to_numpy()
+    eligible_embeddings = profile_embeddings[eligible_indices]
+
+    query_emb = embedder.encode(
+        [query],
+        convert_to_numpy=True,
+        normalize_embeddings=True,
+    ).astype(np.float32)
+
+    similarities = np.dot(query_emb, eligible_embeddings.T)[0]
+
+    display_count = max(1, min(display_count, len(eligible_indices)))
+    best_local_indices = np.argsort(-similarities)[:display_count]
+
+    cards = []
+
+    if activity_filter == "Max age in days" and custom_days > 0:
+        filter_label = f"active in last {custom_days} days"
+    elif activity_filter == "Has known activity date":
+        filter_label = "with known public activity date"
+    elif activity_filter == "No known activity date":
+        filter_label = "with no known public activity date"
+    else:
+        filter_label = "no date filter"
+
+    header = f"""
+    <div class="search-summary">
+        <b>{len(eligible):,}</b> eligible profiles · showing top <b>{display_count}</b> · <b>{filter_label}</b>
+    </div>
+    """.replace(",", " ")
+
+    cards.append(header)
+
+    for rank, local_idx in enumerate(best_local_indices, start=1):
+        global_idx = eligible_indices[local_idx]
+        row = profile_df.iloc[global_idx]
+        score = float(similarities[local_idx])
+        cards.append(render_profile_card(row, score, rank))
+
+    has_more = display_count < len(eligible_indices)
+
+    if not has_more:
+        cards.append("""
+        <div class="empty-state">
+            All eligible profiles are already displayed.
+        </div>
+        """)
+
+    return "\n".join(cards), display_count, has_more
+
+
+def search_hf_atlas(query, activity_filter, custom_days):
+    results_html, display_count, has_more = build_search_results(
+        query=query,
+        activity_filter=activity_filter,
+        custom_days=custom_days,
+        display_count=10,
+    )
+
+    more_button_update = gr.update(visible=has_more)
+
+    return results_html, display_count, more_button_update
+
+
+def search_more_hf_atlas(query, activity_filter, custom_days, display_count):
+    if display_count is None:
+        display_count = 10
+
+    display_count = int(display_count)
+
+    if display_count <= 0:
+        display_count = 10
+
+    new_display_count = display_count + 10
+
+    results_html, final_display_count, has_more = build_search_results(
+        query=query,
+        activity_filter=activity_filter,
+        custom_days=custom_days,
+        display_count=new_display_count,
+    )
+
+    more_button_update = gr.update(visible=has_more)
+
+    return results_html, final_display_count, more_button_update
+
+
+css = """
+body {
+    background: radial-gradient(circle at top left, #172554 0%, #020617 35%, #020617 100%);
+    font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, sans-serif;
+}
+
+.gradio-container {
+    max-width: 980px !important;
+}
+
+#title {
+    font-size: 3.1em;
+    font-weight: 900;
+    text-align: center;
+    color: #e0f2fe;
+    text-shadow: 0 0 18px rgba(56, 189, 248, 0.45);
+    margin-bottom: 0;
+}
+
+#subtitle {
+    color: #bae6fd;
+    text-align: center;
+    margin-top: 0.6em;
+    margin-bottom: 2.2em;
+    font-size: 1.15em;
+}
+
+textarea {
+    background: rgba(15, 23, 42, 0.92) !important;
+    border: 1px solid rgba(125, 211, 252, 0.55) !important;
+    color: #e0f2fe !important;
+    border-radius: 18px !important;
+}
+
+input, select {
+    background: rgba(15, 23, 42, 0.90) !important;
+    border: 1px solid rgba(56, 189, 248, 0.35) !important;
+    color: #e0f2fe !important;
+}
+
+button {
+    background: linear-gradient(135deg, #38bdf8, #818cf8) !important;
+    border: none !important;
+    color: #020617 !important;
+    font-weight: 900 !important;
+    font-size: 1.08em !important;
+    border-radius: 18px !important;
+    box-shadow: 0 0 24px rgba(56, 189, 248, 0.35);
+}
+
+button:hover {
+    transform: scale(1.015);
+    box-shadow: 0 0 34px rgba(129, 140, 248, 0.55);
+}
+
+.result-card {
+    background: linear-gradient(135deg, rgba(15, 23, 42, 0.96), rgba(30, 41, 59, 0.88));
+    border: 1px solid rgba(125, 211, 252, 0.35);
+    border-radius: 24px;
+    padding: 22px;
+    margin: 18px 0;
+    box-shadow: 0 16px 44px rgba(0, 0, 0, 0.34);
+}
+
+.result-topline {
+    display: flex;
+    justify-content: space-between;
+    gap: 16px;
+    align-items: flex-start;
+}
+
+.rank {
+    color: #7dd3fc;
+    font-size: 0.92em;
+    font-weight: 800;
+    letter-spacing: 0.08em;
+}
+
+.username {
+    color: #e0f2fe !important;
+    font-size: 1.55em;
+    font-weight: 900;
+    text-decoration: none !important;
+}
+
+.username:hover {
+    color: #38bdf8 !important;
+    text-decoration: underline !important;
+}
+
+.fullname {
+    color: #cbd5e1;
+    margin-top: 4px;
+    font-size: 0.98em;
+}
+
+.score {
+    color: #020617;
+    background: linear-gradient(135deg, #67e8f9, #a5b4fc);
+    padding: 9px 13px;
+    border-radius: 999px;
+    font-weight: 900;
+    min-width: 90px;
+    text-align: center;
+}
+
+.details {
+    color: #cbd5e1;
+    background: rgba(2, 6, 23, 0.38);
+    border-left: 3px solid rgba(56, 189, 248, 0.65);
+    padding: 12px 14px;
+    margin-top: 16px;
+    border-radius: 14px;
+}
+
+.interests {
+    margin-top: 16px;
+    color: #e0f2fe;
+    line-height: 1.55;
+}
+
+.label {
+    color: #7dd3fc;
+    font-weight: 900;
+    margin-bottom: 6px;
+    text-transform: uppercase;
+    letter-spacing: 0.08em;
+    font-size: 0.78em;
+}
+
+.stats {
+    display: flex;
+    flex-wrap: wrap;
+    gap: 10px;
+    margin-top: 18px;
+}
+
+.stats span {
+    background: rgba(14, 165, 233, 0.10);
+    color: #bae6fd;
+    border: 1px solid rgba(125, 211, 252, 0.22);
+    border-radius: 999px;
+    padding: 7px 11px;
+    font-size: 0.92em;
+}
+
+.search-summary {
+    color: #bae6fd;
+    text-align: center;
+    background: rgba(15, 23, 42, 0.7);
+    border: 1px solid rgba(125, 211, 252, 0.25);
+    border-radius: 18px;
+    padding: 12px;
+    margin-bottom: 18px;
+}
+
+.empty-state {
+    text-align: center;
+    color: #bae6fd;
+    background: rgba(15, 23, 42, 0.75);
+    border: 1px solid rgba(125, 211, 252, 0.25);
+    border-radius: 18px;
+    padding: 24px;
+    margin-top: 16px;
+}
+
+.more-button-wrap {
+    margin-top: 10px;
+    margin-bottom: 30px;
+}
+
+.nebula {
+    position: fixed;
+    inset: 0;
+    pointer-events: none;
+    z-index: 0;
+    opacity: 0.40;
+    background:
+        radial-gradient(circle at 20% 20%, rgba(56,189,248,0.24), transparent 28%),
+        radial-gradient(circle at 80% 30%, rgba(129,140,248,0.22), transparent 30%),
+        radial-gradient(circle at 50% 80%, rgba(14,165,233,0.16), transparent 26%);
+}
+"""
+
+
+with gr.Blocks(css=css, theme=gr.themes.Base(), title="HF Atlas Explorer") as demo:
+    gr.HTML('<div class="nebula"></div>')
+    gr.HTML('<h1 id="title">🧭 HF Atlas Explorer</h1>')
+    gr.HTML(
+        '<p id="subtitle">Search Hugging Face profiles by AI/ML interests and filter by public activity.</p>'
+    )
+
+    query = gr.Textbox(
+        label="Search query",
+        placeholder="e.g. diffusion models, biomedical NLP, reinforcement learning, graph neural networks, robotics...",
+        lines=4,
+    )
+
+    with gr.Row():
+        activity_filter = gr.Dropdown(
+            choices=[
+                "No filter",
+                "Max age in days",
+                "Has known activity date",
+                "No known activity date",
+            ],
+            value="Max age in days",
+            label="Last public activity filter",
+        )
+
+        custom_days = gr.Number(
+            label="Max last_seen age in days, 0 = no limit",
+            value=365,
+            precision=0,
+        )
+
+    display_count_state = gr.State(value=0)
+
+    submit_btn = gr.Button("🔎 Search HF Atlas")
+
+    output = gr.HTML()
+
+    with gr.Row(elem_classes=["more-button-wrap"]):
+        more_btn = gr.Button("➕ Afficher plus", visible=False)
+
+    submit_btn.click(
+        fn=search_hf_atlas,
+        inputs=[query, activity_filter, custom_days],
+        outputs=[output, display_count_state, more_btn],
+    )
+
+    more_btn.click(
+        fn=search_more_hf_atlas,
+        inputs=[query, activity_filter, custom_days, display_count_state],
+        outputs=[output, display_count_state, more_btn],
+    )
+
+
+
+demo.launch()