| import html |
| from datetime import datetime, timezone |
|
|
| import gradio as gr |
| import numpy as np |
| import pandas as pd |
| from sentence_transformers import SentenceTransformer |
|
|
|
|
| DATA_PATH = "users_ai_ml_interests_only.parquet" |
| EMBEDDINGS_PATH = "embeddings_ai_ml_interests.npy" |
| MODEL_NAME = "BAAI/bge-small-en-v1.5" |
|
|
|
|
| profile_df = pd.read_parquet(DATA_PATH) |
| profile_embeddings = np.load(EMBEDDINGS_PATH).astype(np.float32) |
|
|
| print(f"✅ Loaded {len(profile_df)} HF Atlas profiles from parquet") |
| print(f"✅ Loaded embeddings: {profile_embeddings.shape}") |
|
|
| if len(profile_df) != profile_embeddings.shape[0]: |
| raise ValueError( |
| f"Parquet / embeddings mismatch: {len(profile_df)} rows vs {profile_embeddings.shape[0]} embeddings" |
| ) |
|
|
|
|
| def detect_username_col(df): |
| for col in ["user", "username", "namespace"]: |
| if col in df.columns: |
| return col |
| return None |
|
|
|
|
| USERNAME_COL = detect_username_col(profile_df) |
|
|
| if USERNAME_COL is None: |
| raise ValueError("No username column found. Expected one of: user, username, namespace") |
|
|
|
|
| def normalize_embeddings_if_needed(embeddings): |
| norms = np.linalg.norm(embeddings, axis=1, keepdims=True) |
| norms = np.where(norms == 0, 1.0, norms) |
| return embeddings / norms |
|
|
|
|
| profile_embeddings = normalize_embeddings_if_needed(profile_embeddings) |
|
|
| embedder = SentenceTransformer(MODEL_NAME) |
|
|
|
|
| def safe_text(value, default=""): |
| if value is None: |
| return default |
|
|
| try: |
| if pd.isna(value): |
| return default |
| except Exception: |
| pass |
|
|
| text = str(value).strip() |
|
|
| if text.lower() in {"nan", "none", "null"}: |
| return default |
|
|
| return text |
|
|
|
|
| def parse_last_seen(value): |
| text = safe_text(value) |
|
|
| if not text: |
| return pd.NaT |
|
|
| return pd.to_datetime(text, errors="coerce", utc=True) |
|
|
|
|
| def prepare_dates(): |
| if "last_seen_all_repo" not in profile_df.columns: |
| profile_df["_last_seen_dt"] = pd.NaT |
| print("⚠️ Column last_seen_all_repo not found. Date filter will only work as no-filter.") |
| return |
|
|
| profile_df["_last_seen_dt"] = profile_df["last_seen_all_repo"].map(parse_last_seen) |
|
|
| known = int(profile_df["_last_seen_dt"].notna().sum()) |
| unknown = int(profile_df["_last_seen_dt"].isna().sum()) |
|
|
| print(f"🕒 Known last_seen_all_repo dates: {known}") |
| print(f"🕳️ Unknown last_seen_all_repo dates: {unknown}") |
|
|
| if known > 0: |
| print(f"🕒 Min last_seen: {profile_df['_last_seen_dt'].min()}") |
| print(f"🕒 Max last_seen: {profile_df['_last_seen_dt'].max()}") |
|
|
|
|
| prepare_dates() |
|
|
|
|
| def filter_by_activity(df, activity_filter, custom_days): |
| if "_last_seen_dt" not in df.columns: |
| return df |
|
|
| custom_days = int(custom_days) if custom_days else 0 |
|
|
| if activity_filter == "No filter": |
| return df |
|
|
| if activity_filter == "Has known activity date": |
| return df[df["_last_seen_dt"].notna()] |
|
|
| if activity_filter == "No known activity date": |
| return df[df["_last_seen_dt"].isna()] |
|
|
| if activity_filter == "Max age in days": |
| if custom_days <= 0: |
| return df |
|
|
| now = pd.Timestamp(datetime.now(timezone.utc)) |
| cutoff = now - pd.Timedelta(days=custom_days) |
|
|
| return df[df["_last_seen_dt"].notna() & (df["_last_seen_dt"] >= cutoff)] |
|
|
| return df |
|
|
|
|
| def format_number(value): |
| text = safe_text(value, "0") |
|
|
| try: |
| number = int(float(text)) |
| return f"{number:,}".replace(",", " ") |
| except Exception: |
| return html.escape(text) |
|
|
|
|
| def format_date(value): |
| text = safe_text(value) |
|
|
| if not text: |
| return "unknown" |
|
|
| try: |
| dt = pd.to_datetime(text, errors="coerce", utc=True) |
| if pd.isna(dt): |
| return "unknown" |
| return dt.strftime("%Y-%m-%d") |
| except Exception: |
| return html.escape(text) |
|
|
|
|
| def truncate(text, max_len=900): |
| text = safe_text(text) |
|
|
| if len(text) <= max_len: |
| return text |
|
|
| return text[:max_len].rsplit(" ", 1)[0] + "..." |
|
|
|
|
| def get_profile_url(row): |
| if "atlas_request_url" in row and safe_text(row["atlas_request_url"]): |
| return ( |
| safe_text(row["atlas_request_url"]) |
| .replace("/api/users/", "/") |
| .replace("/overview", "") |
| ) |
|
|
| username = safe_text(row[USERNAME_COL]) |
| return f"https://huggingface.co/{username}" |
|
|
|
|
| def render_profile_card(row, score, rank): |
| username = safe_text(row[USERNAME_COL], "unknown") |
| fullname = safe_text(row.get("fullname", ""), "") |
| details = safe_text(row.get("details", ""), "") |
| ai_ml_interests = truncate(row.get("ai_ml_interests", ""), 800) |
| last_seen = format_date(row.get("last_seen_all_repo", "")) |
|
|
| num_models = format_number(row.get("numModels", row.get("n_models", 0))) |
| num_datasets = format_number(row.get("numDatasets", row.get("n_datasets", 0))) |
| num_spaces = format_number(row.get("numSpaces", row.get("n_spaces", 0))) |
| followers = format_number(row.get("numFollowers", 0)) |
| likes = format_number(row.get("numLikes", row.get("numUpvotes", 0))) |
|
|
| url = get_profile_url(row) |
|
|
| title = html.escape(username) |
| fullname_html = html.escape(fullname) if fullname else "—" |
| details_html = html.escape(truncate(details, 300)) if details else "" |
| interests_html = html.escape(ai_ml_interests).replace("\n", "<br>") |
|
|
| extra_details = "" |
| if details_html: |
| extra_details = f""" |
| <div class="details">{details_html}</div> |
| """ |
|
|
| return f""" |
| <div class="result-card"> |
| <div class="result-topline"> |
| <div> |
| <div class="rank">#{rank}</div> |
| <a class="username" href="{url}" target="_blank">{title}</a> |
| <div class="fullname">{fullname_html}</div> |
| </div> |
| <div class="score">{score * 100:.2f}%</div> |
| </div> |
| |
| {extra_details} |
| |
| <div class="interests"> |
| <div class="label">AI/ML interests</div> |
| <div>{interests_html}</div> |
| </div> |
| |
| <div class="stats"> |
| <span>🧠 Models: <b>{num_models}</b></span> |
| <span>📚 Datasets: <b>{num_datasets}</b></span> |
| <span>🚀 Spaces: <b>{num_spaces}</b></span> |
| <span>❤️ Likes: <b>{likes}</b></span> |
| <span>👥 Followers: <b>{followers}</b></span> |
| <span>🕒 Last seen: <b>{last_seen}</b></span> |
| </div> |
| </div> |
| """ |
|
|
|
|
| def build_search_results(query, activity_filter, custom_days, display_count): |
| query = safe_text(query) |
|
|
| if not query: |
| return """ |
| <div class="empty-state"> |
| Describe an AI/ML topic, research area, tool, model family, or technical interest. |
| </div> |
| """, 0, False |
|
|
| custom_days = int(custom_days) if custom_days else 0 |
| display_count = int(display_count) |
|
|
| eligible = filter_by_activity(profile_df, activity_filter, custom_days) |
|
|
| print("FILTER:", activity_filter, "DAYS:", custom_days, "ELIGIBLE:", len(eligible)) |
|
|
| if "_last_seen_dt" in eligible.columns and len(eligible) > 0: |
| known_eligible = eligible[eligible["_last_seen_dt"].notna()] |
| if len(known_eligible) > 0: |
| print("ELIGIBLE MIN LAST_SEEN:", known_eligible["_last_seen_dt"].min()) |
| print("ELIGIBLE MAX LAST_SEEN:", known_eligible["_last_seen_dt"].max()) |
|
|
| if len(eligible) == 0: |
| return """ |
| <div class="empty-state"> |
| No profile found for this activity filter. |
| </div> |
| """, display_count, False |
|
|
| eligible_indices = eligible.index.to_numpy() |
| eligible_embeddings = profile_embeddings[eligible_indices] |
|
|
| query_emb = embedder.encode( |
| [query], |
| convert_to_numpy=True, |
| normalize_embeddings=True, |
| ).astype(np.float32) |
|
|
| similarities = np.dot(query_emb, eligible_embeddings.T)[0] |
|
|
| display_count = max(1, min(display_count, len(eligible_indices))) |
| best_local_indices = np.argsort(-similarities)[:display_count] |
|
|
| cards = [] |
|
|
| if activity_filter == "Max age in days" and custom_days > 0: |
| filter_label = f"active in last {custom_days} days" |
| elif activity_filter == "Has known activity date": |
| filter_label = "with known public activity date" |
| elif activity_filter == "No known activity date": |
| filter_label = "with no known public activity date" |
| else: |
| filter_label = "no date filter" |
|
|
| header = f""" |
| <div class="search-summary"> |
| <b>{len(eligible):,}</b> eligible profiles · showing top <b>{display_count}</b> · <b>{filter_label}</b> |
| </div> |
| """.replace(",", " ") |
|
|
| cards.append(header) |
|
|
| for rank, local_idx in enumerate(best_local_indices, start=1): |
| global_idx = eligible_indices[local_idx] |
| row = profile_df.iloc[global_idx] |
| score = float(similarities[local_idx]) |
| cards.append(render_profile_card(row, score, rank)) |
|
|
| has_more = display_count < len(eligible_indices) |
|
|
| if not has_more: |
| cards.append(""" |
| <div class="empty-state"> |
| All eligible profiles are already displayed. |
| </div> |
| """) |
|
|
| return "\n".join(cards), display_count, has_more |
|
|
|
|
| def search_hf_atlas(query, activity_filter, custom_days): |
| results_html, display_count, has_more = build_search_results( |
| query=query, |
| activity_filter=activity_filter, |
| custom_days=custom_days, |
| display_count=10, |
| ) |
|
|
| more_button_update = gr.update(visible=has_more) |
|
|
| return results_html, display_count, more_button_update |
|
|
|
|
| def search_more_hf_atlas(query, activity_filter, custom_days, display_count): |
| if display_count is None: |
| display_count = 10 |
|
|
| display_count = int(display_count) |
|
|
| if display_count <= 0: |
| display_count = 10 |
|
|
| new_display_count = display_count + 10 |
|
|
| results_html, final_display_count, has_more = build_search_results( |
| query=query, |
| activity_filter=activity_filter, |
| custom_days=custom_days, |
| display_count=new_display_count, |
| ) |
|
|
| more_button_update = gr.update(visible=has_more) |
|
|
| return results_html, final_display_count, more_button_update |
|
|
|
|
| css = """ |
| body { |
| background: radial-gradient(circle at top left, #172554 0%, #020617 35%, #020617 100%); |
| font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, sans-serif; |
| } |
| |
| .gradio-container { |
| max-width: 980px !important; |
| } |
| |
| #title { |
| font-size: 3.1em; |
| font-weight: 900; |
| text-align: center; |
| color: #e0f2fe; |
| text-shadow: 0 0 18px rgba(56, 189, 248, 0.45); |
| margin-bottom: 0; |
| } |
| |
| #subtitle { |
| color: #bae6fd; |
| text-align: center; |
| margin-top: 0.6em; |
| margin-bottom: 2.2em; |
| font-size: 1.15em; |
| } |
| |
| textarea { |
| background: rgba(15, 23, 42, 0.92) !important; |
| border: 1px solid rgba(125, 211, 252, 0.55) !important; |
| color: #e0f2fe !important; |
| border-radius: 18px !important; |
| } |
| |
| input, select { |
| background: rgba(15, 23, 42, 0.90) !important; |
| border: 1px solid rgba(56, 189, 248, 0.35) !important; |
| color: #e0f2fe !important; |
| } |
| |
| button { |
| background: linear-gradient(135deg, #38bdf8, #818cf8) !important; |
| border: none !important; |
| color: #020617 !important; |
| font-weight: 900 !important; |
| font-size: 1.08em !important; |
| border-radius: 18px !important; |
| box-shadow: 0 0 24px rgba(56, 189, 248, 0.35); |
| } |
| |
| button:hover { |
| transform: scale(1.015); |
| box-shadow: 0 0 34px rgba(129, 140, 248, 0.55); |
| } |
| |
| .result-card { |
| background: linear-gradient(135deg, rgba(15, 23, 42, 0.96), rgba(30, 41, 59, 0.88)); |
| border: 1px solid rgba(125, 211, 252, 0.35); |
| border-radius: 24px; |
| padding: 22px; |
| margin: 18px 0; |
| box-shadow: 0 16px 44px rgba(0, 0, 0, 0.34); |
| } |
| |
| .result-topline { |
| display: flex; |
| justify-content: space-between; |
| gap: 16px; |
| align-items: flex-start; |
| } |
| |
| .rank { |
| color: #7dd3fc; |
| font-size: 0.92em; |
| font-weight: 800; |
| letter-spacing: 0.08em; |
| } |
| |
| .username { |
| color: #e0f2fe !important; |
| font-size: 1.55em; |
| font-weight: 900; |
| text-decoration: none !important; |
| } |
| |
| .username:hover { |
| color: #38bdf8 !important; |
| text-decoration: underline !important; |
| } |
| |
| .fullname { |
| color: #cbd5e1; |
| margin-top: 4px; |
| font-size: 0.98em; |
| } |
| |
| .score { |
| color: #020617; |
| background: linear-gradient(135deg, #67e8f9, #a5b4fc); |
| padding: 9px 13px; |
| border-radius: 999px; |
| font-weight: 900; |
| min-width: 90px; |
| text-align: center; |
| } |
| |
| .details { |
| color: #cbd5e1; |
| background: rgba(2, 6, 23, 0.38); |
| border-left: 3px solid rgba(56, 189, 248, 0.65); |
| padding: 12px 14px; |
| margin-top: 16px; |
| border-radius: 14px; |
| } |
| |
| .interests { |
| margin-top: 16px; |
| color: #e0f2fe; |
| line-height: 1.55; |
| } |
| |
| .label { |
| color: #7dd3fc; |
| font-weight: 900; |
| margin-bottom: 6px; |
| text-transform: uppercase; |
| letter-spacing: 0.08em; |
| font-size: 0.78em; |
| } |
| |
| .stats { |
| display: flex; |
| flex-wrap: wrap; |
| gap: 10px; |
| margin-top: 18px; |
| } |
| |
| .stats span { |
| background: rgba(14, 165, 233, 0.10); |
| color: #bae6fd; |
| border: 1px solid rgba(125, 211, 252, 0.22); |
| border-radius: 999px; |
| padding: 7px 11px; |
| font-size: 0.92em; |
| } |
| |
| .search-summary { |
| color: #bae6fd; |
| text-align: center; |
| background: rgba(15, 23, 42, 0.7); |
| border: 1px solid rgba(125, 211, 252, 0.25); |
| border-radius: 18px; |
| padding: 12px; |
| margin-bottom: 18px; |
| } |
| |
| .empty-state { |
| text-align: center; |
| color: #bae6fd; |
| background: rgba(15, 23, 42, 0.75); |
| border: 1px solid rgba(125, 211, 252, 0.25); |
| border-radius: 18px; |
| padding: 24px; |
| margin-top: 16px; |
| } |
| |
| .more-button-wrap { |
| margin-top: 10px; |
| margin-bottom: 30px; |
| } |
| |
| .nebula { |
| position: fixed; |
| inset: 0; |
| pointer-events: none; |
| z-index: 0; |
| opacity: 0.40; |
| background: |
| radial-gradient(circle at 20% 20%, rgba(56,189,248,0.24), transparent 28%), |
| radial-gradient(circle at 80% 30%, rgba(129,140,248,0.22), transparent 30%), |
| radial-gradient(circle at 50% 80%, rgba(14,165,233,0.16), transparent 26%); |
| } |
| """ |
|
|
|
|
| with gr.Blocks(title="HF Atlas Explorer") as demo: |
| gr.HTML('<div class="nebula"></div>') |
| gr.HTML('<h1 id="title"> 🌐 HF Atlas Explorer</h1>') |
| gr.HTML( |
| '<p id="subtitle">Search Hugging Face profiles by AI/ML interests and filter by public activity.</p>' |
| ) |
|
|
| query = gr.Textbox( |
| label="Search query", |
| placeholder="e.g. diffusion models, biomedical NLP, reinforcement learning, graph neural networks, robotics...", |
| lines=4, |
| ) |
|
|
| with gr.Row(): |
| activity_filter = gr.Dropdown( |
| choices=[ |
| "No filter", |
| "Max age in days", |
| "Has known activity date", |
| "No known activity date", |
| ], |
| value="Max age in days", |
| label="Last public activity filter", |
| ) |
|
|
| custom_days = gr.Number( |
| label="Max last_seen age in days, 0 = no limit", |
| value=365, |
| precision=0, |
| ) |
|
|
| display_count_state = gr.State(value=0) |
|
|
| submit_btn = gr.Button("🔎 Search HF Atlas") |
|
|
| output = gr.HTML() |
|
|
| with gr.Row(elem_classes=["more-button-wrap"]): |
| more_btn = gr.Button("➕ Show more", visible=False) |
|
|
| submit_btn.click( |
| fn=search_hf_atlas, |
| inputs=[query, activity_filter, custom_days], |
| outputs=[output, display_count_state, more_btn], |
| ) |
|
|
| more_btn.click( |
| fn=search_more_hf_atlas, |
| inputs=[query, activity_filter, custom_days, display_count_state], |
| outputs=[output, display_count_state, more_btn], |
| ) |
|
|
|
|
| demo.launch(css=css, theme=gr.themes.Base()) |
|
|
