""" HF Dataset Explorer — view ANY Hugging Face dataset, even when the official dataset-viewer is unavailable / stuck / not-yet-built. Run: python app.py (add --share for a public link) """ from __future__ import annotations import argparse import json import os import tempfile import traceback import gradio as gr import pandas as pd import backend as B STATE_DEFAULT = {"dataset": "", "config": "", "split": "", "df": None, "offset": 0, "total": None, "token": ""} EXAMPLES = [ "stanfordnlp/imdb", "openai/gsm8k", "cais/mmlu", "HuggingFaceFW/fineweb", "lmms-lab/COCO-Caption", ] # --------------------------------------------------------------------------- # # tab 1: connect # --------------------------------------------------------------------------- # def connect(raw_id, token): try: ds = B.normalize_id(raw_id) except B.LoadError as e: return (gr.update(), gr.update(), f"### ❌ {e}", "", gr.update(choices=[]), {}) lines = [f"## `{ds}`", ""] try: info = B.repo_info(ds, token) lines += [ f"- **Downloads (30d):** {info.get('downloads', '?'):,} · **Likes:** {info.get('likes', '?')}", f"- **Modified:** {str(info.get('lastModified', ''))[:19]}", f"- **Tags:** {', '.join(info.get('tags', [])[:14]) or '—'}", f"- **Gated:** {info.get('gated', False)} · **Private:** {info.get('private', False)}", ] except Exception as e: # noqa: BLE001 lines.append(f"> ⚠️ repo metadata unavailable: {e}") ok, detail = B.viewer_status(ds, token) lines += ["", f"- **Official viewer:** {'✅ ready' if ok else '⛔ NOT ready'} — `{detail}`", f" {'' if ok else '→ no problem, this app falls back to parquet / streaming / raw files.'}"] try: mapping = B.discover_splits(ds, token) except Exception as e: # noqa: BLE001 mapping = {"default": ["train"]} lines.append(f"> ⚠️ split discovery fell back to guesses: {e}") configs = list(mapping) first_cfg = configs[0] splits = mapping[first_cfg] lines.append(f"- **Configs:** {len(configs)} · **Splits in `{first_cfg}`:** {', '.join(splits)}") # file tree try: files = B.list_files(ds, token) tab = pd.DataFrame([{"path": f["path"], "size": f.get("size") or (f.get("lfs") or {}).get("size") or 0} for f in files if f.get("type") == "file"]) if not tab.empty: tab["size"] = tab["size"].map(lambda n: f"{n/1e6:,.2f} MB" if n else "—") tabular = [f["path"] for f in files if f.get("type") == "file" and f["path"].lower().endswith(B.TABULAR_EXT)] except Exception: tab, tabular = pd.DataFrame(), [] return (gr.update(choices=configs, value=first_cfg), gr.update(choices=splits, value=splits[0] if splits else None), "\n".join(lines), B.readme(ds, token), gr.update(choices=tabular[:500], value=tabular[0] if tabular else None), {"dataset": ds, "mapping": mapping, "files": tab.to_dict("records")}) def on_config_change(cfg, meta): splits = (meta or {}).get("mapping", {}).get(cfg, ["train"]) return gr.update(choices=splits, value=splits[0] if splits else None) def file_table(meta): recs = (meta or {}).get("files", []) return pd.DataFrame(recs) if recs else pd.DataFrame({"path": [], "size": []}) # --------------------------------------------------------------------------- # # tab 2: rows # --------------------------------------------------------------------------- # def _render(res: B.LoadResult, state, maxlen, cols_pick=None): df = res.df state = dict(state or {}) state.update(df=df, offset=res.offset, total=res.total_rows, strategy=res.strategy) media = B.detect_media_columns(df) total = f"{res.total_rows:,}" if res.total_rows else "unknown" banner = (f"**Strategy:** `{res.strategy}` · **rows {res.offset:,}–{res.offset+len(df)-1:,}** " f"of {total} · **{len(df.columns)} columns**" + (f"\n\n_{' · '.join(res.notes)}_" if res.notes else "") + (f"\n\n🖼️ image cols: `{media['image']}`" if media["image"] else "") + (f" 🔊 audio cols: `{media['audio']}`" if media["audio"] else "")) keep = [c for c in (cols_pick or []) if c in df.columns] show = df[keep] if keep else df return (B.display_frame(show, maxlen), banner, state, gr.update(choices=[str(c) for c in df.columns], value=[str(c) for c in df.columns]), gr.update(maximum=max(1, len(df) - 1), value=0)) def load_rows(dataset_meta, cfg, split, offset, limit, token, strategy, maxlen, state): ds = (dataset_meta or {}).get("dataset", "") if not ds: return (pd.DataFrame(), "### ❌ Connect to a dataset first (tab 1).", state, gr.update(), gr.update()) try: res = B.load_rows(ds, cfg, split, int(offset), int(limit), token, strategy) except Exception as e: # noqa: BLE001 return (pd.DataFrame(), f"### ❌ Could not load rows\n```\n{e}\n```", state, gr.update(), gr.update()) state = dict(state or {}); state["dataset"] = ds return _render(res, state, int(maxlen)) def page(delta, dataset_meta, cfg, split, offset, limit, token, strategy, maxlen, state): new_off = max(0, int(offset) + delta * int(limit)) out = load_rows(dataset_meta, cfg, split, new_off, limit, token, strategy, maxlen, state) return (*out, new_off) def _state_df(state) -> pd.DataFrame: df = (state or {}).get("df") return df if isinstance(df, pd.DataFrame) else pd.DataFrame() def apply_filter(query, regex_col, regex, state, maxlen): df = _state_df(state) if df.empty: return pd.DataFrame(), "No data loaded." note = [] try: if query.strip(): df = df.query(query, engine="python") note.append(f"query `{query}`") if regex.strip() and regex_col: df = df[df[regex_col].astype(str).str.contains(regex, case=False, regex=True, na=False)] note.append(f"regex `{regex}` on `{regex_col}`") except Exception as e: # noqa: BLE001 return pd.DataFrame(), f"❌ filter error: `{e}`" return B.display_frame(df, int(maxlen)), f"**{len(df):,} matching rows** ({' + '.join(note) or 'no filter'})" def pick_columns(cols, state, maxlen): df = _state_df(state) if df.empty: return pd.DataFrame() keep = [c for c in df.columns if str(c) in set(cols or [])] return B.display_frame(df[keep] if keep else df, int(maxlen)) # --------------------------------------------------------------------------- # # tab 3: record inspector # --------------------------------------------------------------------------- # def inspect(idx, state): df = _state_df(state) if df.empty: return {}, "No data loaded.", None, None i = max(0, min(int(idx), len(df) - 1)) row = df.iloc[i] payload = {str(k): B.json_safe(v) for k, v in row.items()} md = [f"### Row {i} (absolute #{(state or {}).get('offset', 0) + i})"] for k, v in row.items(): s = B.stringify(v, 4000) md.append(f"**{k}**\n\n```\n{s}\n```") img = aud = None media = B.detect_media_columns(df) if media["image"]: img = B.media_to_display(row[media["image"][0]]) if media["audio"]: aud = B.media_to_display(row[media["audio"][0]]) return payload, "\n\n".join(md), img, aud # --------------------------------------------------------------------------- # # tab 4: stats # --------------------------------------------------------------------------- # def profile(state): df = _state_df(state) if df.empty: return pd.DataFrame(), pd.DataFrame(), "No data loaded." rows = [] for c in df.columns: s = df[c] flat = s.map(lambda v: B.stringify(v, 10_000)) rows.append({ "column": str(c), "dtype": str(s.dtype), "nulls": int(s.isna().sum()), "null %": round(100 * s.isna().mean(), 2), "unique": int(flat.nunique()), "mean len": round(flat.str.len().mean(), 1), "max len": int(flat.str.len().max()), "example": B.stringify(s.dropna().iloc[0], 80) if s.notna().any() else "", }) schema = pd.DataFrame(rows) num = df.select_dtypes("number") desc = num.describe().T.reset_index().rename(columns={"index": "column"}) if not num.empty \ else pd.DataFrame({"info": ["no numeric columns in this page"]}) # label-ish distribution md = ["### Value distributions (low-cardinality columns)"] for c in df.columns: flat = df[c].map(lambda v: B.stringify(v, 60)) if 1 < flat.nunique() <= 25: vc = flat.value_counts() md.append(f"**{c}**") md += [f"- `{k}` — {v} ({100*v/len(flat):.1f}%)" for k, v in vc.items()] if len(md) == 1: md.append("_none found on this page._") return schema, desc, "\n".join(md) # --------------------------------------------------------------------------- # # tab 5: media gallery # --------------------------------------------------------------------------- # def gallery(state, col, n): df = _state_df(state) if df.empty: return [], "No data loaded." media = B.detect_media_columns(df) col = col or (media["image"][0] if media["image"] else None) if not col or col not in df.columns: return [], "No image-like column detected on this page." items = [] for i, v in df[col].head(int(n)).items(): d = B.media_to_display(v) if d is not None: items.append((d, f"#{i}")) return items, f"{len(items)} image(s) from `{col}`" def media_choices(state): df = _state_df(state) if df.empty: return gr.update(choices=[]) m = B.detect_media_columns(df) return gr.update(choices=m["image"] + m["audio"], value=(m["image"] or m["audio"] or [None])[0]) # --------------------------------------------------------------------------- # # tab 6: raw files # --------------------------------------------------------------------------- # def read_raw(dataset_meta, path, offset, limit, token, maxlen): ds = (dataset_meta or {}).get("dataset", "") if not ds or not path: return pd.DataFrame(), "Pick a file first." try: res = B.read_raw_file(ds, path, token, int(limit), int(offset)) except Exception as e: # noqa: BLE001 return pd.DataFrame(), f"❌ {e}" return (B.display_frame(res.df, int(maxlen)), f"**{res.strategy}** · {len(res.df)} rows" + (f" of {res.total_rows:,}" if res.total_rows else "")) # --------------------------------------------------------------------------- # # tab 7: export # --------------------------------------------------------------------------- # def export(state, fmt): df = _state_df(state) if df.empty: return None, "Nothing to export." name = f"{(state or {}).get('dataset','hf')}".replace("/", "__") or "hf_export" d = tempfile.mkdtemp() path = os.path.join(d, f"{name}.{ {'CSV':'csv','JSON Lines':'jsonl','Parquet':'parquet'}[fmt] }") try: if fmt == "CSV": B.display_frame(df, 10**6).to_csv(path, index=False) elif fmt == "JSON Lines": with open(path, "w", encoding="utf-8") as fh: for _, r in df.iterrows(): fh.write(json.dumps({str(k): B.json_safe(v) for k, v in r.items()}, ensure_ascii=False) + "\n") else: df.to_parquet(path, index=False) except Exception as e: # noqa: BLE001 return None, f"❌ {e}" return path, f"✅ exported {len(df):,} rows" def code_snippet(meta, cfg, split, strategy): ds = (meta or {}).get("dataset", "your/dataset") return f'''```python # Reproduce this page in your own code from datasets import load_dataset # 1) streaming — works even when the Hub viewer is not ready ds = load_dataset("{ds}", {json.dumps(cfg) if cfg else "None"}, split="{split}", streaming=True) for i, ex in zip(range(5), ds): print(ex) # 2) direct parquet (random access, no dataset script executed) import pandas as pd df = pd.read_parquet( "hf://datasets/{ds}@refs/convert/parquet/{cfg or 'default'}/{split}/0000.parquet" ) ```''' # --------------------------------------------------------------------------- # # UI # --------------------------------------------------------------------------- # CSS = """ .small-note {font-size: 0.85rem; opacity: .75} footer {display:none !important} """ with gr.Blocks(title="HF Dataset Explorer") as demo: meta = gr.State({}) state = gr.State(dict(STATE_DEFAULT)) gr.Markdown( "# 🤗 HF Dataset Explorer\n" "View **any** Hugging Face dataset — even when the official dataset viewer " "is not ready. Falls back automatically: `viewer API → parquet → streaming → raw files`." ) with gr.Row(): dataset_id = gr.Textbox(label="Dataset id or URL", value="stanfordnlp/imdb", placeholder="user/name, or paste a huggingface.co URL", scale=4) token = gr.Textbox(label="HF token (optional, for private/gated)", type="password", scale=2) connect_btn = gr.Button("Connect", variant="primary", scale=1) gr.Examples(EXAMPLES, inputs=dataset_id, label="Try") with gr.Row(): config = gr.Dropdown(label="Config", choices=[], interactive=True, allow_custom_value=True) split = gr.Dropdown(label="Split", choices=[], interactive=True, allow_custom_value=True) strategy = gr.Dropdown(label="Loader", value="Auto (try everything)", choices=["Auto (try everything)", *B.STRATEGIES]) limit = gr.Slider(5, 500, 50, step=5, label="Rows per page") offset = gr.Number(0, label="Offset", precision=0) maxlen = gr.Slider(50, 2000, 300, step=50, label="Cell truncation (chars)") info_md = gr.Markdown() with gr.Tabs(): with gr.Tab("📋 Rows"): with gr.Row(): load_btn = gr.Button("Load rows", variant="primary") prev_btn = gr.Button("◀ Prev page") next_btn = gr.Button("Next page ▶") banner = gr.Markdown() cols = gr.Dropdown(label="Visible columns", multiselect=True, choices=[], interactive=True) table = gr.Dataframe(wrap=True, interactive=False, max_height=650) with gr.Tab("🔎 Filter & search"): gr.Markdown("Filters run on the **currently loaded page** (client-side, instant).") with gr.Row(): query = gr.Textbox(label="pandas query", placeholder="label == 1 and len(text) > 500") regex_col = gr.Dropdown(label="Regex column", choices=[], allow_custom_value=True) regex = gr.Textbox(label="Regex / substring", placeholder="great|terrible") filter_btn = gr.Button("Apply filter", variant="primary") filter_note = gr.Markdown() ftable = gr.Dataframe(wrap=True, interactive=False, max_height=600) with gr.Tab("🔬 Record inspector"): row_idx = gr.Slider(0, 1, 0, step=1, label="Row on this page") with gr.Row(): with gr.Column(scale=3): rec_md = gr.Markdown() with gr.Column(scale=2): rec_json = gr.JSON(label="Raw record") rec_img = gr.Image(label="Image field", height=260) rec_aud = gr.Audio(label="Audio field") with gr.Tab("📊 Stats"): stats_btn = gr.Button("Profile this page", variant="primary") schema_tbl = gr.Dataframe(label="Schema & quality", wrap=True, interactive=False) num_tbl = gr.Dataframe(label="Numeric describe()", interactive=False) dist_md = gr.Markdown() with gr.Tab("🖼️ Media"): with gr.Row(): media_col = gr.Dropdown(label="Media column", choices=[], allow_custom_value=True) n_media = gr.Slider(1, 100, 24, step=1, label="How many") media_btn = gr.Button("Render", variant="primary") media_note = gr.Markdown() gal = gr.Gallery(columns=6, height=520, object_fit="contain") with gr.Tab("🗂️ Repo files"): gr.Markdown("Bypass everything and read a file straight out of the repo.") files_tbl = gr.Dataframe(interactive=False, max_height=280) with gr.Row(): file_pick = gr.Dropdown(label="Tabular file", choices=[], allow_custom_value=True, scale=3) raw_off = gr.Number(0, label="Offset", precision=0) raw_lim = gr.Slider(5, 500, 50, step=5, label="Rows") raw_btn = gr.Button("Read file", variant="primary") raw_note = gr.Markdown() raw_tbl = gr.Dataframe(wrap=True, interactive=False, max_height=520) with gr.Tab("📄 README"): readme_md = gr.Markdown() with gr.Tab("⬇️ Export & code"): with gr.Row(): fmt = gr.Radio(["CSV", "JSON Lines", "Parquet"], value="CSV", label="Format") exp_btn = gr.Button("Export current page", variant="primary") exp_note = gr.Markdown() exp_file = gr.File(label="Download") snippet = gr.Markdown() # ---------------- wiring ---------------- # connect_btn.click(connect, [dataset_id, token], [config, split, info_md, readme_md, file_pick, meta]) \ .then(file_table, meta, files_tbl) \ .then(code_snippet, [meta, config, split, strategy], snippet) dataset_id.submit(connect, [dataset_id, token], [config, split, info_md, readme_md, file_pick, meta]) \ .then(file_table, meta, files_tbl) config.change(on_config_change, [config, meta], split) load_inputs = [meta, config, split, offset, limit, token, strategy, maxlen, state] load_btn.click(load_rows, load_inputs, [table, banner, state, cols, row_idx]) \ .then(lambda s: gr.update(choices=[str(c) for c in _state_df(s).columns]), state, regex_col) \ .then(media_choices, state, media_col) \ .then(code_snippet, [meta, config, split, strategy], snippet) next_btn.click(lambda *a: page(1, *a), load_inputs, [table, banner, state, cols, row_idx, offset]) prev_btn.click(lambda *a: page(-1, *a), load_inputs, [table, banner, state, cols, row_idx, offset]) cols.change(pick_columns, [cols, state, maxlen], table) filter_btn.click(apply_filter, [query, regex_col, regex, state, maxlen], [ftable, filter_note]) row_idx.change(inspect, [row_idx, state], [rec_json, rec_md, rec_img, rec_aud]) stats_btn.click(profile, state, [schema_tbl, num_tbl, dist_md]) media_btn.click(gallery, [state, media_col, n_media], [gal, media_note]) raw_btn.click(read_raw, [meta, file_pick, raw_off, raw_lim, token, maxlen], [raw_tbl, raw_note]) exp_btn.click(export, [state, fmt], [exp_file, exp_note]) if __name__ == "__main__": p = argparse.ArgumentParser() p.add_argument("--share", action="store_true") p.add_argument("--port", type=int, default=7860) a = p.parse_args() kwargs = dict(server_name="0.0.0.0", server_port=a.port, share=a.share, show_error=True) try: # gradio >= 6 moved theme/css to launch() demo.queue(default_concurrency_limit=4).launch( theme=gr.themes.Soft(), css=CSS, **kwargs) except TypeError: demo.queue(default_concurrency_limit=4).launch(**kwargs)