| """A "Did You Know?" card shuffler for the wikimedia/wiki-dyk dataset. |
| |
| Shows one random hook at a time: the featured article's image, the fact |
| (rendered from content_body), its topics, and a link to the article exactly |
| as it stood the day it was featured. Filter by language and topic. |
| |
| Data is read straight from the published dataset on the Hub. Set DYK_LOCAL to |
| a directory of dyk-{lang}.parquet files to run against local outputs instead. |
| """ |
|
|
| import html |
| import os |
| import random |
| import re |
| from collections import Counter |
| from urllib.parse import quote |
|
|
| import gradio as gr |
| import pandas as pd |
|
|
| DATASET = "wikimedia/wiki-dyk" |
| LANGS = ["en", "fr", "de"] |
| TOP_TOPICS = 200 |
| ALL_TOPICS = "All topics" |
|
|
| _cache: dict[str, dict] = {} |
|
|
|
|
| def _data_url(lang: str) -> str: |
| local = os.environ.get("DYK_LOCAL") |
| if local: |
| return f"{local}/dyk-{lang}.parquet" |
| return f"hf://datasets/{DATASET}/{lang}/dyk-{lang}.parquet" |
|
|
|
|
| def _as_list(value) -> list: |
| """linked_pages / article_topics arrive as numpy arrays or None.""" |
| return [] if value is None else list(value) |
|
|
|
|
| def _clean_int(value): |
| return None if value is None or pd.isna(value) else int(value) |
|
|
|
|
| def load(lang: str) -> dict: |
| """Load and normalize one language, cached. Builds a frequency-ranked |
| topic -> row-indices index for O(1) filtered sampling.""" |
| if lang in _cache: |
| return _cache[lang] |
|
|
| df = pd.read_parquet( |
| _data_url(lang), |
| columns=["day", "content_body", "linked_pages"], |
| ) |
|
|
| records = [] |
| counter: Counter = Counter() |
| for day, body, linked in zip( |
| df["day"], df["content_body"], df["linked_pages"], strict=True |
| ): |
| pages = [] |
| topics_all: set[str] = set() |
| for page in _as_list(linked): |
| topics = [ |
| topic["title"] |
| for topic in _as_list(page["article_topics"]) |
| if topic is not None and topic["title"] |
| ] |
| pages.append( |
| { |
| "page_title": page["page_title"], |
| "short_description": page["short_description"], |
| "page_image": page["page_image"], |
| "rev_id": _clean_int(page["rev_id"]), |
| "prominent": bool(page["prominent"]), |
| "topics": topics, |
| } |
| ) |
| topics_all.update(topics) |
| counter.update(topics_all) |
| records.append( |
| {"day": day, "content_body": body, "pages": pages, "topics": topics_all} |
| ) |
|
|
| top = {title for title, _ in counter.most_common(TOP_TOPICS)} |
| index: dict[str, list[int]] = {title: [] for title in top} |
| for i, record in enumerate(records): |
| for title in record["topics"] & top: |
| index[title].append(i) |
| choices = sorted( |
| ((title.replace("_", " "), title) for title in top), key=lambda c: c[0].lower() |
| ) |
|
|
| _cache[lang] = { |
| "records": records, |
| "all_idx": list(range(len(records))), |
| "topic_index": index, |
| "topic_choices": choices, |
| } |
| return _cache[lang] |
|
|
|
|
| def _featured(pages: list[dict]) -> dict | None: |
| """The article the card centres on: a prominent one, preferring one that |
| has an image; fall back to any imaged page, then the first page.""" |
| prominent = [p for p in pages if p["prominent"]] |
| for pool in (prominent, pages): |
| imaged = [p for p in pool if p["page_image"]] |
| if imaged: |
| return imaged[0] |
| if prominent: |
| return prominent[0] |
| return pages[0] if pages else None |
|
|
|
|
| def _image_url(lang: str, filename: str) -> str: |
| return ( |
| f"https://{lang}.wikipedia.org/wiki/Special:FilePath/" |
| f"{quote(filename)}?width=400" |
| ) |
|
|
|
|
| def _wiki_url(lang: str, title: str) -> str: |
| return f"https://{lang}.wikipedia.org/wiki/{quote(title.replace(' ', '_'))}" |
|
|
|
|
| def _rewrite_links(lang: str, body: str) -> str: |
| """content_body links are relative (href="./Title").""" |
| return re.sub( |
| r'href="\./([^"]*)"', |
| lambda m: ( |
| f'href="https://{lang}.wikipedia.org/wiki/{m.group(1)}" ' |
| f'target="_blank" rel="noopener"' |
| ), |
| body, |
| ) |
|
|
|
|
| def _fmt_day(day) -> str | None: |
| if day is None or pd.isna(day): |
| return None |
| d = pd.Timestamp(day) |
| return f"{d.day} {d:%B %Y}" |
|
|
|
|
| CARD_CSS = """ |
| <style> |
| .dyk-card { max-width: 640px; margin: 0 auto; padding: 1.25rem; |
| border: 1px solid rgba(128,128,128,.3); border-radius: 16px; } |
| .dyk-head { display: flex; gap: 1rem; align-items: flex-start; } |
| .dyk-img { width: 120px; height: 120px; object-fit: cover; border-radius: 12px; |
| flex-shrink: 0; } |
| .dyk-title { font-size: 1.1rem; font-weight: 600; } |
| .dyk-desc { opacity: .7; font-size: .9rem; } |
| .dyk-fact { margin: 1rem 0; font-size: 1.15rem; line-height: 1.5; } |
| .dyk-chips { display: flex; flex-wrap: wrap; gap: .4rem; margin-bottom: .75rem; } |
| .dyk-chip { font-size: .8rem; padding: .15rem .6rem; border-radius: 999px; |
| border: 1px solid rgba(128,128,128,.3); opacity: .85; } |
| .dyk-prov { font-size: .85rem; opacity: .7; } |
| .dyk-prov a { color: inherit; } |
| </style> |
| """ |
|
|
|
|
| def render_card(lang: str, record: dict | None) -> str: |
| if record is None: |
| return f"{CARD_CSS}<div class='dyk-card'>No facts to show.</div>" |
|
|
| featured = _featured(record["pages"]) |
| head = "" |
| if featured: |
| img = "" |
| if featured["page_image"]: |
| img = ( |
| f'<img class="dyk-img" alt="" ' |
| f'src="{_image_url(lang, featured["page_image"])}">' |
| ) |
| title = html.escape(featured["page_title"]) |
| desc = html.escape(featured["short_description"] or "") |
| head = ( |
| f'<div class="dyk-head">{img}<div>' |
| f'<div class="dyk-title">' |
| f'<a href="{_wiki_url(lang, featured["page_title"])}" ' |
| f'target="_blank" rel="noopener" style="color:inherit">{title}</a></div>' |
| f'<div class="dyk-desc">{desc}</div></div></div>' |
| ) |
|
|
| chips = "" |
| if featured and featured["topics"]: |
| chips = ( |
| '<div class="dyk-chips">' |
| + "".join( |
| f'<span class="dyk-chip">#{html.escape(t.replace("_", " "))}</span>' |
| for t in featured["topics"] |
| ) |
| + "</div>" |
| ) |
|
|
| prov = "" |
| day = _fmt_day(record["day"]) |
| if day: |
| prov = f"Featured on {day}" |
| if featured and featured["rev_id"]: |
| link = f"https://{lang}.wikipedia.org/w/index.php?oldid={featured['rev_id']}" |
| sep = " · " if prov else "" |
| prov += ( |
| f'{sep}<a href="{link}" target="_blank" rel="noopener">' |
| f"see the article as it was then →</a>" |
| ) |
| prov = f'<div class="dyk-prov">{prov}</div>' if prov else "" |
|
|
| fact = _rewrite_links(lang, record["content_body"]) |
| return ( |
| f'{CARD_CSS}<div class="dyk-card">{head}' |
| f'<div class="dyk-fact">{fact}</div>{chips}{prov}</div>' |
| ) |
|
|
|
|
| def shuffle(lang: str, topic: str) -> str: |
| data = load(lang) |
| if topic and topic != ALL_TOPICS: |
| candidates = data["topic_index"].get(topic, []) |
| else: |
| candidates = data["all_idx"] |
| record = data["records"][random.choice(candidates)] if candidates else None |
| return render_card(lang, record) |
|
|
|
|
| def switch_language(lang: str): |
| data = load(lang) |
| choices = [(ALL_TOPICS, ALL_TOPICS), *data["topic_choices"]] |
| return ( |
| gr.update(choices=choices, value=ALL_TOPICS), |
| shuffle(lang, ALL_TOPICS), |
| ) |
|
|
|
|
| with gr.Blocks(title="Wikipedia — Did You Know?") as demo: |
| gr.Markdown( |
| "# 🎲 Wikipedia — Did You Know?\n" |
| "Random facts from the " |
| "[wikimedia/wiki-dyk](https://huggingface.co/datasets/wikimedia/wiki-dyk) " |
| "dataset — the *Did you know* snippets Wikipedia featured on its Main Page." |
| ) |
| with gr.Row(): |
| lang = gr.Radio(LANGS, value="en", label="Language") |
| topic = gr.Dropdown( |
| choices=[(ALL_TOPICS, ALL_TOPICS)], |
| value=ALL_TOPICS, |
| label="Topic", |
| filterable=True, |
| ) |
| card = gr.HTML() |
| gr.Button("Another fact →", variant="primary").click(shuffle, [lang, topic], card) |
|
|
| lang.change(switch_language, lang, [topic, card]) |
| topic.change(shuffle, [lang, topic], card) |
| demo.load(switch_language, lang, [topic, card]) |
|
|
| if __name__ == "__main__": |
| demo.launch() |
|
|
