Spaces:
Sleeping
Sleeping
File size: 5,222 Bytes
d8db673 2214565 d8db673 2214565 d8db673 2214565 d8db673 2214565 d8db673 2214565 d8db673 2214565 d8db673 2214565 d8db673 2214565 d8db673 2214565 4d4cb57 2214565 d8db673 2214565 4d4cb57 2214565 4d4cb57 2214565 4d4cb57 2214565 4d4cb57 2214565 4d4cb57 2214565 d8db673 4d4cb57 d8db673 2214565 d8db673 4d4cb57 2214565 d8db673 4d4cb57 2214565 d8db673 2214565 d8db673 2214565 4d4cb57 2214565 4d4cb57 2214565 4d4cb57 2214565 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 | from __future__ import annotations
"""Serve short glosses from the local Kaikki dump baked into atlas.sqlite."""
import json
import sqlite3
MAX_SENSES = 4
MAX_GLOSSES = 4
def _trim_senses(raw: object) -> list[dict]:
if isinstance(raw, str):
try:
raw = json.loads(raw)
except json.JSONDecodeError:
return []
if not isinstance(raw, list):
return []
senses: list[dict] = []
for block in raw:
if not isinstance(block, dict):
continue
pos = str(block.get("pos") or "Sense").strip() or "Sense"
glosses: list[str] = []
for g in block.get("glosses") or []:
text = " ".join(str(g).split()).strip()
if text:
glosses.append(text)
if len(glosses) >= MAX_GLOSSES:
break
if glosses:
senses.append({"pos": pos, "glosses": glosses})
if len(senses) >= MAX_SENSES:
break
return senses
def _candidate_langs(lang: str, iso_639_3: str | None, lang_meta: dict[str, dict] | None) -> list[str]:
out: list[str] = []
key = (lang or "").strip()
if key:
out.append(key)
folded = key.casefold()
if folded != key:
out.append(folded)
iso = (iso_639_3 or "").strip().casefold()
if lang_meta and iso:
for lk, meta in lang_meta.items():
if (meta.get("iso_639_3") or "").casefold() == iso:
out.append(lk)
if iso:
out.append(iso)
seen: set[str] = set()
uniq: list[str] = []
for c in out:
if c and c not in seen:
seen.add(c)
uniq.append(c)
return uniq
def lookup_definitions(
conn: sqlite3.Connection,
term: str,
lang: str = "",
ety: str | None = None,
iso_639_3: str | None = None,
lang_meta: dict[str, dict] | None = None,
) -> dict:
term = (term or "").strip()
if not term:
return {"term": term, "senses": [], "error": "empty_term", "source": "kaikki"}
candidates = _candidate_langs(lang, iso_639_3, lang_meta)
row = None
matched_lang = None
matched_ety = None
for cand in candidates:
if ety is not None:
row = conn.execute(
"SELECT lang, ety, senses_json FROM definitions WHERE term = ? AND lang = ? AND ety = ?",
(term, cand, ety),
).fetchone()
else:
row = conn.execute(
"""
SELECT lang, ety, senses_json FROM definitions
WHERE term = ? AND lang = ?
ORDER BY (ety = '') DESC, ety
LIMIT 1
""",
(term, cand),
).fetchone()
if row is not None:
matched_lang = row["lang"] if isinstance(row, sqlite3.Row) else row[0]
matched_ety = row["ety"] if isinstance(row, sqlite3.Row) else row[1]
break
fallback = False
if row is None:
rows = conn.execute(
"SELECT lang, ety, senses_json FROM definitions WHERE term = ? ORDER BY lang, ety LIMIT 8",
(term,),
).fetchall()
if rows:
prefer = {c.casefold() for c in candidates}
chosen = None
for r in rows:
rlang = r["lang"] if isinstance(r, sqlite3.Row) else r[0]
if rlang.casefold() in prefer or rlang.casefold() == "english":
chosen = r
break
row = chosen or rows[0]
matched_lang = row["lang"] if isinstance(row, sqlite3.Row) else row[0]
matched_ety = row["ety"] if isinstance(row, sqlite3.Row) else row[1]
fallback = bool(candidates) and matched_lang not in candidates
if row is None:
return {
"term": term,
"lang": lang,
"ety": ety or "",
"matched_code": None,
"senses": [],
"fallback": False,
"source": "kaikki",
"error": "not_found",
}
senses_json = row["senses_json"] if isinstance(row, sqlite3.Row) else row[2]
senses = _trim_senses(senses_json)
return {
"term": term,
"lang": lang,
"ety": matched_ety or "",
"matched_code": matched_lang,
"senses": senses,
"fallback": fallback,
"source": "kaikki",
"error": None if senses else "no_senses",
}
def fetch_definitions(
term: str,
lang: str = "",
ety: str | None = None,
iso_639_3: str | None = None,
*,
atlas=None,
) -> dict:
"""Lookup glosses from the baked index. ``atlas`` must be a loaded Atlas."""
if atlas is None or getattr(atlas, "conn", None) is None:
return {
"term": term,
"lang": lang,
"ety": ety or "",
"matched_code": None,
"senses": [],
"fallback": False,
"source": "kaikki",
"error": "index_unavailable",
}
return lookup_definitions(
atlas.conn,
term=term,
lang=lang,
ety=ety,
iso_639_3=iso_639_3,
lang_meta=getattr(atlas, "lang_meta", None),
)
|