import html from datetime import datetime, timezone import gradio as gr import numpy as np import pandas as pd from sentence_transformers import SentenceTransformer DATA_PATH = "users_ai_ml_interests_only.parquet" EMBEDDINGS_PATH = "embeddings_ai_ml_interests.npy" MODEL_NAME = "BAAI/bge-small-en-v1.5" profile_df = pd.read_parquet(DATA_PATH) profile_embeddings = np.load(EMBEDDINGS_PATH).astype(np.float32) print(f"โœ… Loaded {len(profile_df)} HF Atlas profiles from parquet") print(f"โœ… Loaded embeddings: {profile_embeddings.shape}") if len(profile_df) != profile_embeddings.shape[0]: raise ValueError( f"Parquet / embeddings mismatch: {len(profile_df)} rows vs {profile_embeddings.shape[0]} embeddings" ) def detect_username_col(df): for col in ["user", "username", "namespace"]: if col in df.columns: return col return None USERNAME_COL = detect_username_col(profile_df) if USERNAME_COL is None: raise ValueError("No username column found. Expected one of: user, username, namespace") def normalize_embeddings_if_needed(embeddings): norms = np.linalg.norm(embeddings, axis=1, keepdims=True) norms = np.where(norms == 0, 1.0, norms) return embeddings / norms profile_embeddings = normalize_embeddings_if_needed(profile_embeddings) embedder = SentenceTransformer(MODEL_NAME) def safe_text(value, default=""): if value is None: return default try: if pd.isna(value): return default except Exception: pass text = str(value).strip() if text.lower() in {"nan", "none", "null"}: return default return text def parse_last_seen(value): text = safe_text(value) if not text: return pd.NaT return pd.to_datetime(text, errors="coerce", utc=True) def prepare_dates(): if "last_seen_all_repo" not in profile_df.columns: profile_df["_last_seen_dt"] = pd.NaT print("โš ๏ธ Column last_seen_all_repo not found. Date filter will only work as no-filter.") return profile_df["_last_seen_dt"] = profile_df["last_seen_all_repo"].map(parse_last_seen) known = int(profile_df["_last_seen_dt"].notna().sum()) unknown = int(profile_df["_last_seen_dt"].isna().sum()) print(f"๐Ÿ•’ Known last_seen_all_repo dates: {known}") print(f"๐Ÿ•ณ๏ธ Unknown last_seen_all_repo dates: {unknown}") if known > 0: print(f"๐Ÿ•’ Min last_seen: {profile_df['_last_seen_dt'].min()}") print(f"๐Ÿ•’ Max last_seen: {profile_df['_last_seen_dt'].max()}") prepare_dates() def filter_by_activity(df, activity_filter, custom_days): if "_last_seen_dt" not in df.columns: return df custom_days = int(custom_days) if custom_days else 0 if activity_filter == "No filter": return df if activity_filter == "Has known activity date": return df[df["_last_seen_dt"].notna()] if activity_filter == "No known activity date": return df[df["_last_seen_dt"].isna()] if activity_filter == "Max age in days": if custom_days <= 0: return df now = pd.Timestamp(datetime.now(timezone.utc)) cutoff = now - pd.Timedelta(days=custom_days) return df[df["_last_seen_dt"].notna() & (df["_last_seen_dt"] >= cutoff)] return df def format_number(value): text = safe_text(value, "0") try: number = int(float(text)) return f"{number:,}".replace(",", " ") except Exception: return html.escape(text) def format_date(value): text = safe_text(value) if not text: return "unknown" try: dt = pd.to_datetime(text, errors="coerce", utc=True) if pd.isna(dt): return "unknown" return dt.strftime("%Y-%m-%d") except Exception: return html.escape(text) def truncate(text, max_len=900): text = safe_text(text) if len(text) <= max_len: return text return text[:max_len].rsplit(" ", 1)[0] + "..." def get_profile_url(row): if "atlas_request_url" in row and safe_text(row["atlas_request_url"]): return ( safe_text(row["atlas_request_url"]) .replace("/api/users/", "/") .replace("/overview", "") ) username = safe_text(row[USERNAME_COL]) return f"https://huggingface.co/{username}" def render_profile_card(row, score, rank): username = safe_text(row[USERNAME_COL], "unknown") fullname = safe_text(row.get("fullname", ""), "") details = safe_text(row.get("details", ""), "") ai_ml_interests = truncate(row.get("ai_ml_interests", ""), 800) last_seen = format_date(row.get("last_seen_all_repo", "")) num_models = format_number(row.get("numModels", row.get("n_models", 0))) num_datasets = format_number(row.get("numDatasets", row.get("n_datasets", 0))) num_spaces = format_number(row.get("numSpaces", row.get("n_spaces", 0))) followers = format_number(row.get("numFollowers", 0)) likes = format_number(row.get("numLikes", row.get("numUpvotes", 0))) url = get_profile_url(row) title = html.escape(username) fullname_html = html.escape(fullname) if fullname else "โ€”" details_html = html.escape(truncate(details, 300)) if details else "" interests_html = html.escape(ai_ml_interests).replace("\n", "
") extra_details = "" if details_html: extra_details = f"""
{details_html}
""" return f"""
#{rank}
{title}
{fullname_html}
{score * 100:.2f}%
{extra_details}
AI/ML interests
{interests_html}
๐Ÿง  Models: {num_models} ๐Ÿ“š Datasets: {num_datasets} ๐Ÿš€ Spaces: {num_spaces} โค๏ธ Likes: {likes} ๐Ÿ‘ฅ Followers: {followers} ๐Ÿ•’ Last seen: {last_seen}
""" def build_search_results(query, activity_filter, custom_days, display_count): query = safe_text(query) if not query: return """
Describe an AI/ML topic, research area, tool, model family, or technical interest.
""", 0, False custom_days = int(custom_days) if custom_days else 0 display_count = int(display_count) eligible = filter_by_activity(profile_df, activity_filter, custom_days) print("FILTER:", activity_filter, "DAYS:", custom_days, "ELIGIBLE:", len(eligible)) if "_last_seen_dt" in eligible.columns and len(eligible) > 0: known_eligible = eligible[eligible["_last_seen_dt"].notna()] if len(known_eligible) > 0: print("ELIGIBLE MIN LAST_SEEN:", known_eligible["_last_seen_dt"].min()) print("ELIGIBLE MAX LAST_SEEN:", known_eligible["_last_seen_dt"].max()) if len(eligible) == 0: return """
No profile found for this activity filter.
""", display_count, False eligible_indices = eligible.index.to_numpy() eligible_embeddings = profile_embeddings[eligible_indices] query_emb = embedder.encode( [query], convert_to_numpy=True, normalize_embeddings=True, ).astype(np.float32) similarities = np.dot(query_emb, eligible_embeddings.T)[0] display_count = max(1, min(display_count, len(eligible_indices))) best_local_indices = np.argsort(-similarities)[:display_count] cards = [] if activity_filter == "Max age in days" and custom_days > 0: filter_label = f"active in last {custom_days} days" elif activity_filter == "Has known activity date": filter_label = "with known public activity date" elif activity_filter == "No known activity date": filter_label = "with no known public activity date" else: filter_label = "no date filter" header = f"""
{len(eligible):,} eligible profiles ยท showing top {display_count} ยท {filter_label}
""".replace(",", " ") cards.append(header) for rank, local_idx in enumerate(best_local_indices, start=1): global_idx = eligible_indices[local_idx] row = profile_df.iloc[global_idx] score = float(similarities[local_idx]) cards.append(render_profile_card(row, score, rank)) has_more = display_count < len(eligible_indices) if not has_more: cards.append("""
All eligible profiles are already displayed.
""") return "\n".join(cards), display_count, has_more def search_hf_atlas(query, activity_filter, custom_days): results_html, display_count, has_more = build_search_results( query=query, activity_filter=activity_filter, custom_days=custom_days, display_count=10, ) more_button_update = gr.update(visible=has_more) return results_html, display_count, more_button_update def search_more_hf_atlas(query, activity_filter, custom_days, display_count): if display_count is None: display_count = 10 display_count = int(display_count) if display_count <= 0: display_count = 10 new_display_count = display_count + 10 results_html, final_display_count, has_more = build_search_results( query=query, activity_filter=activity_filter, custom_days=custom_days, display_count=new_display_count, ) more_button_update = gr.update(visible=has_more) return results_html, final_display_count, more_button_update css = """ body { background: radial-gradient(circle at top left, #172554 0%, #020617 35%, #020617 100%); font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, sans-serif; } .gradio-container { max-width: 980px !important; } #title { font-size: 3.1em; font-weight: 900; text-align: center; color: #e0f2fe; text-shadow: 0 0 18px rgba(56, 189, 248, 0.45); margin-bottom: 0; } #subtitle { color: #bae6fd; text-align: center; margin-top: 0.6em; margin-bottom: 2.2em; font-size: 1.15em; } textarea { background: rgba(15, 23, 42, 0.92) !important; border: 1px solid rgba(125, 211, 252, 0.55) !important; color: #e0f2fe !important; border-radius: 18px !important; } input, select { background: rgba(15, 23, 42, 0.90) !important; border: 1px solid rgba(56, 189, 248, 0.35) !important; color: #e0f2fe !important; } button { background: linear-gradient(135deg, #38bdf8, #818cf8) !important; border: none !important; color: #020617 !important; font-weight: 900 !important; font-size: 1.08em !important; border-radius: 18px !important; box-shadow: 0 0 24px rgba(56, 189, 248, 0.35); } button:hover { transform: scale(1.015); box-shadow: 0 0 34px rgba(129, 140, 248, 0.55); } .result-card { background: linear-gradient(135deg, rgba(15, 23, 42, 0.96), rgba(30, 41, 59, 0.88)); border: 1px solid rgba(125, 211, 252, 0.35); border-radius: 24px; padding: 22px; margin: 18px 0; box-shadow: 0 16px 44px rgba(0, 0, 0, 0.34); } .result-topline { display: flex; justify-content: space-between; gap: 16px; align-items: flex-start; } .rank { color: #7dd3fc; font-size: 0.92em; font-weight: 800; letter-spacing: 0.08em; } .username { color: #e0f2fe !important; font-size: 1.55em; font-weight: 900; text-decoration: none !important; } .username:hover { color: #38bdf8 !important; text-decoration: underline !important; } .fullname { color: #cbd5e1; margin-top: 4px; font-size: 0.98em; } .score { color: #020617; background: linear-gradient(135deg, #67e8f9, #a5b4fc); padding: 9px 13px; border-radius: 999px; font-weight: 900; min-width: 90px; text-align: center; } .details { color: #cbd5e1; background: rgba(2, 6, 23, 0.38); border-left: 3px solid rgba(56, 189, 248, 0.65); padding: 12px 14px; margin-top: 16px; border-radius: 14px; } .interests { margin-top: 16px; color: #e0f2fe; line-height: 1.55; } .label { color: #7dd3fc; font-weight: 900; margin-bottom: 6px; text-transform: uppercase; letter-spacing: 0.08em; font-size: 0.78em; } .stats { display: flex; flex-wrap: wrap; gap: 10px; margin-top: 18px; } .stats span { background: rgba(14, 165, 233, 0.10); color: #bae6fd; border: 1px solid rgba(125, 211, 252, 0.22); border-radius: 999px; padding: 7px 11px; font-size: 0.92em; } .search-summary { color: #bae6fd; text-align: center; background: rgba(15, 23, 42, 0.7); border: 1px solid rgba(125, 211, 252, 0.25); border-radius: 18px; padding: 12px; margin-bottom: 18px; } .empty-state { text-align: center; color: #bae6fd; background: rgba(15, 23, 42, 0.75); border: 1px solid rgba(125, 211, 252, 0.25); border-radius: 18px; padding: 24px; margin-top: 16px; } .more-button-wrap { margin-top: 10px; margin-bottom: 30px; } .nebula { position: fixed; inset: 0; pointer-events: none; z-index: 0; opacity: 0.40; background: radial-gradient(circle at 20% 20%, rgba(56,189,248,0.24), transparent 28%), radial-gradient(circle at 80% 30%, rgba(129,140,248,0.22), transparent 30%), radial-gradient(circle at 50% 80%, rgba(14,165,233,0.16), transparent 26%); } """ with gr.Blocks(title="HF Atlas Explorer") as demo: gr.HTML('
') gr.HTML('

๐ŸŒ HF Atlas Explorer

') gr.HTML( '

Search Hugging Face profiles by AI/ML interests and filter by public activity.

' ) query = gr.Textbox( label="Search query", placeholder="e.g. diffusion models, biomedical NLP, reinforcement learning, graph neural networks, robotics...", lines=4, ) with gr.Row(): activity_filter = gr.Dropdown( choices=[ "No filter", "Max age in days", "Has known activity date", "No known activity date", ], value="Max age in days", label="Last public activity filter", ) custom_days = gr.Number( label="Max last_seen age in days, 0 = no limit", value=365, precision=0, ) display_count_state = gr.State(value=0) submit_btn = gr.Button("๐Ÿ”Ž Search HF Atlas") output = gr.HTML() with gr.Row(elem_classes=["more-button-wrap"]): more_btn = gr.Button("โž• Show more", visible=False) submit_btn.click( fn=search_hf_atlas, inputs=[query, activity_filter, custom_days], outputs=[output, display_count_state, more_btn], ) more_btn.click( fn=search_more_hf_atlas, inputs=[query, activity_filter, custom_days, display_count_state], outputs=[output, display_count_state, more_btn], ) demo.launch(css=css, theme=gr.themes.Base())