"""A "Did You Know?" card shuffler for the wikimedia/wiki-dyk dataset.
Shows one random hook at a time: the featured article's image, the fact
(rendered from content_body), its topics, and a link to the article exactly
as it stood the day it was featured. Filter by language and topic.
Data is read straight from the published dataset on the Hub. Set DYK_LOCAL to
a directory of dyk-{lang}.parquet files to run against local outputs instead.
"""
import html
import os
import random
import re
from collections import Counter
from urllib.parse import quote
import gradio as gr
import pandas as pd
DATASET = "wikimedia/wiki-dyk"
LANGS = ["en", "fr", "de"]
TOP_TOPICS = 200 # dropdown cap; the full unique topic list is unusably long
ALL_TOPICS = "All topics"
_cache: dict[str, dict] = {}
def _data_url(lang: str) -> str:
local = os.environ.get("DYK_LOCAL")
if local:
return f"{local}/dyk-{lang}.parquet"
return f"hf://datasets/{DATASET}/{lang}/dyk-{lang}.parquet"
def _as_list(value) -> list:
"""linked_pages / article_topics arrive as numpy arrays or None."""
return [] if value is None else list(value)
def _clean_int(value):
return None if value is None or pd.isna(value) else int(value)
def load(lang: str) -> dict:
"""Load and normalize one language, cached. Builds a frequency-ranked
topic -> row-indices index for O(1) filtered sampling."""
if lang in _cache:
return _cache[lang]
df = pd.read_parquet(
_data_url(lang),
columns=["day", "content_body", "linked_pages"],
)
records = []
counter: Counter = Counter()
for day, body, linked in zip(
df["day"], df["content_body"], df["linked_pages"], strict=True
):
pages = []
topics_all: set[str] = set()
for page in _as_list(linked):
topics = [
topic["title"]
for topic in _as_list(page["article_topics"])
if topic is not None and topic["title"]
]
pages.append(
{
"page_title": page["page_title"],
"short_description": page["short_description"],
"page_image": page["page_image"],
"rev_id": _clean_int(page["rev_id"]),
"prominent": bool(page["prominent"]),
"topics": topics,
}
)
topics_all.update(topics)
counter.update(topics_all)
records.append(
{"day": day, "content_body": body, "pages": pages, "topics": topics_all}
)
top = {title for title, _ in counter.most_common(TOP_TOPICS)}
index: dict[str, list[int]] = {title: [] for title in top}
for i, record in enumerate(records):
for title in record["topics"] & top:
index[title].append(i)
choices = sorted(
((title.replace("_", " "), title) for title in top), key=lambda c: c[0].lower()
)
_cache[lang] = {
"records": records,
"all_idx": list(range(len(records))),
"topic_index": index,
"topic_choices": choices,
}
return _cache[lang]
def _featured(pages: list[dict]) -> dict | None:
"""The article the card centres on: a prominent one, preferring one that
has an image; fall back to any imaged page, then the first page."""
prominent = [p for p in pages if p["prominent"]]
for pool in (prominent, pages):
imaged = [p for p in pool if p["page_image"]]
if imaged:
return imaged[0]
if prominent:
return prominent[0]
return pages[0] if pages else None
def _image_url(lang: str, filename: str) -> str:
return (
f"https://{lang}.wikipedia.org/wiki/Special:FilePath/"
f"{quote(filename)}?width=400"
)
def _wiki_url(lang: str, title: str) -> str:
return f"https://{lang}.wikipedia.org/wiki/{quote(title.replace(' ', '_'))}"
def _rewrite_links(lang: str, body: str) -> str:
"""content_body links are relative (href="./Title")."""
return re.sub(
r'href="\./([^"]*)"',
lambda m: (
f'href="https://{lang}.wikipedia.org/wiki/{m.group(1)}" '
f'target="_blank" rel="noopener"'
),
body,
)
def _fmt_day(day) -> str | None:
if day is None or pd.isna(day):
return None
d = pd.Timestamp(day)
return f"{d.day} {d:%B %Y}"
CARD_CSS = """
"""
def render_card(lang: str, record: dict | None) -> str:
if record is None:
return f"{CARD_CSS}No facts to show.
"
featured = _featured(record["pages"])
head = ""
if featured:
img = ""
if featured["page_image"]:
img = (
f'![]()
'
)
title = html.escape(featured["page_title"])
desc = html.escape(featured["short_description"] or "")
head = (
f''
)
chips = ""
if featured and featured["topics"]:
chips = (
''
+ "".join(
f'#{html.escape(t.replace("_", " "))}'
for t in featured["topics"]
)
+ "
"
)
prov = ""
day = _fmt_day(record["day"])
if day:
prov = f"Featured on {day}"
if featured and featured["rev_id"]:
link = f"https://{lang}.wikipedia.org/w/index.php?oldid={featured['rev_id']}"
sep = " ยท " if prov else ""
prov += (
f'{sep}'
f"see the article as it was then โ"
)
prov = f'{prov}
' if prov else ""
fact = _rewrite_links(lang, record["content_body"])
return (
f'{CARD_CSS}{head}'
f'
{fact}
{chips}{prov}