Fast deploy (index 64717948e3cd)

Dockerfile

@@ -27,7 +27,7 @@ COPY frontend/dist ./frontend/dist
 
 # Pinned prebuilt index artifact (update when republishing the dataset).
 ARG INDEX_URL=https://huggingface.co/datasets/Mongoosetross/reverse-etymology-index/resolve/main/index.tar.zst
-ARG INDEX_SHA256=89aa172cb1685949914166db14532a8784235382adc5a1754a20c208daa12069
+ARG INDEX_SHA256=64717948e3cd47d2b453757c4e692f94321fe407e633a9b2605e5780b699f609
 RUN mkdir -p data/index \
     && curl -fsSL "$INDEX_URL" -o /tmp/index.tar.zst \
     && if [ -n "$INDEX_SHA256" ]; then echo "$INDEX_SHA256  /tmp/index.tar.zst" | sha256sum -c -; fi \

backend/app.py

@@ -86,8 +86,8 @@ def node(term: str, lang: str) -> dict:
 
 @app.get("/api/define")
 def define(term: str, lang: str = "", iso: str | None = None) -> dict:
-    """Short glosses from Wiktionary REST, matched to the atlas language when possible."""
-    return fetch_definitions(term=term, lang=lang, iso_639_3=iso)
+    """Short glosses from the local Kaikki dump (baked into atlas.sqlite)."""
+    return fetch_definitions(term=term, lang=lang, iso_639_3=iso, atlas=atlas)
 
 
 @app.post("/api/tree")

backend/define.py

@@ -1,235 +1,154 @@
 from __future__ import annotations
 
-"""Fetch short glosses from the Wiktionary REST definition API."""
-
-import html
-import re
-import time
-from urllib.parse import quote
-
-import httpx
-
-USER_AGENT = "ReverseEtymologyAtlas/1.0 (https://huggingface.co/spaces/Mongoosetross/reverse-etymology)"
-WIKT_DEF = "https://en.wiktionary.org/api/rest_v1/page/definition/{term}"
-
-# Atlas lang keys / ISO 639-3 -> Wiktionary REST language codes (usually ISO 639-1 / 639-3).
-LANG_TO_WIKT: dict[str, str] = {
-    "english": "en",
-    "spanish": "es",
-    "old spanish": "osp",
-    "french": "fr",
-    "old french": "fro",
-    "middle french": "frm",
-    "latin": "la",
-    "vulgar latin": "la",
-    "italian": "it",
-    "portuguese": "pt",
-    "old portuguese": "roa-opt",
-    "galician": "gl",
-    "catalan": "ca",
-    "romanian": "ro",
-    "german": "de",
-    "dutch": "nl",
-    "greek": "el",
-    "ancient greek": "grc",
-    "russian": "ru",
-    "arabic": "ar",
-    "hebrew": "he",
-    "chinese": "zh",
-    "japanese": "ja",
-    "korean": "ko",
-    "hindi": "hi",
-    "sanskrit": "sa",
-    "proto-indo-european": "ine-pro",
-    "proto-germanic": "gem-pro",
-    "proto-celtic": "cel-pro",
-    "old english": "ang",
-    "middle english": "enm",
-    "scottish gaelic": "gd",
-    "irish": "ga",
-    "welsh": "cy",
-    "polish": "pl",
-    "czech": "cs",
-    "swedish": "sv",
-    "norwegian": "no",
-    "danish": "da",
-    "finnish": "fi",
-    "hungarian": "hu",
-    "turkish": "tr",
-    "persian": "fa",
-    "esperanto": "eo",
-    "tagalog": "tl",
-    "asturian": "ast",
-    "aragonese": "an",
-    "corsican": "co",
-    "occitan": "oc",
-    "basque": "eu",
-    "eng": "en",
-    "spa": "es",
-    "fra": "fr",
-    "fre": "fr",
-    "lat": "la",
-    "ita": "it",
-    "por": "pt",
-    "deu": "de",
-    "ger": "de",
-    "nld": "nl",
-    "dut": "nl",
-    "rus": "ru",
-    "ell": "el",
-    "gre": "el",
-    "arb": "ar",
-    "heb": "he",
-    "cmn": "zh",
-    "jpn": "ja",
-    "kor": "ko",
-    "hin": "hi",
-    "san": "sa",
-    "ang": "ang",
-    "enm": "enm",
-    "fro": "fro",
-    "frm": "frm",
-    "osp": "osp",
-    "ron": "ro",
-    "rum": "ro",
-    "cat": "ca",
-    "glg": "gl",
-    "eus": "eu",
-}
-
-TAG_RE = re.compile(r"<[^>]+>")
-NESTED_LIST_RE = re.compile(r"<(ol|ul)\b[^>]*>.*?</\1>", re.I | re.S)
-SPACE_RE = re.compile(r"\s+")
-
-_cache: dict[str, tuple[float, dict]] = {}
-_CACHE_TTL = 3600.0
-
-
-def strip_markup(raw: str) -> str:
-    text = NESTED_LIST_RE.sub(" ", raw or "")
-    text = TAG_RE.sub(" ", text)
-    text = html.unescape(text)
-    return SPACE_RE.sub(" ", text).strip(" \t\n\r-;,:")
-
-
-def wikt_codes_for(lang: str, iso_639_3: str | None = None) -> list[str]:
-    codes: list[str] = []
-    key = (lang or "").strip().casefold()
-    if key in LANG_TO_WIKT:
-        codes.append(LANG_TO_WIKT[key])
-    iso = (iso_639_3 or "").strip().casefold()
-    if iso and iso in LANG_TO_WIKT:
-        codes.append(LANG_TO_WIKT[iso])
-    if iso and len(iso) == 3 and iso not in codes:
-        # Many Wiktionary keys are ISO 639-3 for historical lects.
-        codes.append(iso)
-    if key and key not in codes:
-        codes.append(key)
-    # de-dupe preserving order
-    seen: set[str] = set()
-    out: list[str] = []
-    for c in codes:
-        if c and c not in seen:
-            seen.add(c)
-            out.append(c)
-    return out
+"""Serve short glosses from the local Kaikki dump baked into atlas.sqlite."""
 
+import json
+import sqlite3
 
-def _parse_entries(payload: dict, prefer: list[str]) -> tuple[str | None, list[dict]]:
-    if not isinstance(payload, dict) or not payload:
-        return None, []
-    chosen = None
-    for code in prefer:
-        if code in payload:
-            chosen = code
-            break
-    if chosen is None:
-        # Prefer English glosses as a readable fallback, else first language.
-        if "en" in payload:
-            chosen = "en"
-        else:
-            chosen = next(iter(payload.keys()))
+MAX_SENSES = 4
+MAX_GLOSSES = 4
+
+
+def _trim_senses(raw: object) -> list[dict]:
+    if isinstance(raw, str):
+        try:
+            raw = json.loads(raw)
+        except json.JSONDecodeError:
+            return []
+    if not isinstance(raw, list):
+        return []
     senses: list[dict] = []
-    for block in payload.get(chosen) or []:
+    for block in raw:
         if not isinstance(block, dict):
             continue
-        pos = (block.get("partOfSpeech") or block.get("partOfSpeech") or "").strip() or "Sense"
+        pos = str(block.get("pos") or "Sense").strip() or "Sense"
         glosses: list[str] = []
-        for d in block.get("definitions") or []:
-            if not isinstance(d, dict):
-                continue
-            gloss = strip_markup(str(d.get("definition") or ""))
-            if gloss:
-                glosses.append(gloss)
-            if len(glosses) >= 4:
+        for g in block.get("glosses") or []:
+            text = " ".join(str(g).split()).strip()
+            if text:
+                glosses.append(text)
+            if len(glosses) >= MAX_GLOSSES:
                 break
         if glosses:
             senses.append({"pos": pos, "glosses": glosses})
-        if len(senses) >= 4:
+        if len(senses) >= MAX_SENSES:
             break
-    return chosen, senses
+    return senses
 
 
-def fetch_definitions(term: str, lang: str = "", iso_639_3: str | None = None) -> dict:
+def _candidate_langs(lang: str, iso_639_3: str | None, lang_meta: dict[str, dict] | None) -> list[str]:
+    out: list[str] = []
+    key = (lang or "").strip()
+    if key:
+        out.append(key)
+        folded = key.casefold()
+        if folded != key:
+            out.append(folded)
+    iso = (iso_639_3 or "").strip().casefold()
+    if lang_meta and iso:
+        for lk, meta in lang_meta.items():
+            if (meta.get("iso_639_3") or "").casefold() == iso:
+                out.append(lk)
+    if iso:
+        out.append(iso)
+    seen: set[str] = set()
+    uniq: list[str] = []
+    for c in out:
+        if c and c not in seen:
+            seen.add(c)
+            uniq.append(c)
+    return uniq
+
+
+def lookup_definitions(
+    conn: sqlite3.Connection,
+    term: str,
+    lang: str = "",
+    iso_639_3: str | None = None,
+    lang_meta: dict[str, dict] | None = None,
+) -> dict:
     term = (term or "").strip()
     if not term:
-        return {"term": term, "senses": [], "error": "empty_term"}
-
-    cache_key = f"{term.casefold()}|{lang.casefold()}|{(iso_639_3 or '').casefold()}"
-    now = time.time()
-    hit = _cache.get(cache_key)
-    if hit and now - hit[0] < _CACHE_TTL:
-        return hit[1]
-
-    prefer = wikt_codes_for(lang, iso_639_3)
-    url = WIKT_DEF.format(term=quote(term, safe=""))
-    wiki_lang = prefer[0] if prefer else (lang or "english").replace(" ", "_")
-    wiki_page = f"https://en.wiktionary.org/wiki/{quote(term, safe='')}#{wiki_lang}"
-
-    try:
-        with httpx.Client(timeout=8.0, headers={"User-Agent": USER_AGENT, "Accept": "application/json"}) as client:
-            resp = client.get(url)
-        if resp.status_code == 404:
-            out = {
-                "term": term,
-                "lang": lang,
-                "matched_code": None,
-                "senses": [],
-                "fallback": False,
-                "source": "wiktionary",
-                "url": wiki_page,
-                "error": "not_found",
-            }
-            _cache[cache_key] = (now, out)
-            return out
-        resp.raise_for_status()
-        payload = resp.json()
-    except Exception as exc:  # noqa: BLE001 — surface soft failure to UI
-        out = {
+        return {"term": term, "senses": [], "error": "empty_term", "source": "kaikki"}
+
+    candidates = _candidate_langs(lang, iso_639_3, lang_meta)
+
+    row = None
+    matched_lang = None
+    for cand in candidates:
+        row = conn.execute(
+            "SELECT lang, senses_json FROM definitions WHERE term = ? AND lang = ?",
+            (term, cand),
+        ).fetchone()
+        if row is not None:
+            matched_lang = row["lang"] if isinstance(row, sqlite3.Row) else row[0]
+            break
+
+    fallback = False
+    if row is None:
+        # Any language for this spelling (last resort for mismatched atlas keys).
+        rows = conn.execute(
+            "SELECT lang, senses_json FROM definitions WHERE term = ? ORDER BY lang LIMIT 8",
+            (term,),
+        ).fetchall()
+        if rows:
+            prefer = {c.casefold() for c in candidates}
+            chosen = None
+            for r in rows:
+                rlang = r["lang"] if isinstance(r, sqlite3.Row) else r[0]
+                if rlang.casefold() in prefer or rlang.casefold() == "english":
+                    chosen = r
+                    break
+            row = chosen or rows[0]
+            matched_lang = row["lang"] if isinstance(row, sqlite3.Row) else row[0]
+            fallback = bool(candidates) and matched_lang not in candidates
+
+    if row is None:
+        return {
             "term": term,
             "lang": lang,
             "matched_code": None,
             "senses": [],
             "fallback": False,
-            "source": "wiktionary",
-            "url": wiki_page,
-            "error": f"fetch_failed:{type(exc).__name__}",
+            "source": "kaikki",
+            "error": "not_found",
         }
-        _cache[cache_key] = (now, out)
-        return out
 
-    matched, senses = _parse_entries(payload if isinstance(payload, dict) else {}, prefer)
-    fallback = bool(matched and prefer and matched not in prefer)
-    out = {
+    senses_json = row["senses_json"] if isinstance(row, sqlite3.Row) else row[1]
+    senses = _trim_senses(senses_json)
+    return {
         "term": term,
         "lang": lang,
-        "matched_code": matched,
+        "matched_code": matched_lang,
         "senses": senses,
         "fallback": fallback,
-        "available_langs": sorted((payload or {}).keys())[:24] if isinstance(payload, dict) else [],
-        "source": "wiktionary",
-        "url": wiki_page,
+        "source": "kaikki",
         "error": None if senses else "no_senses",
     }
-    _cache[cache_key] = (now, out)
-    return out
+
+
+def fetch_definitions(
+    term: str,
+    lang: str = "",
+    iso_639_3: str | None = None,
+    *,
+    atlas=None,
+) -> dict:
+    """Lookup glosses from the baked index. ``atlas`` must be a loaded Atlas."""
+    if atlas is None or getattr(atlas, "conn", None) is None:
+        return {
+            "term": term,
+            "lang": lang,
+            "matched_code": None,
+            "senses": [],
+            "fallback": False,
+            "source": "kaikki",
+            "error": "index_unavailable",
+        }
+    return lookup_definitions(
+        atlas.conn,
+        term=term,
+        lang=lang,
+        iso_639_3=iso_639_3,
+        lang_meta=getattr(atlas, "lang_meta", None),
+    )

requirements.space.txt

@@ -4,4 +4,3 @@ uvicorn[standard]>=0.32.0
 numpy>=2.0.0
 orjson>=3.10.0
 pydantic>=2.9.0
-httpx>=0.27.0