Deploy Reverse Etymology Atlas

backend/atlas.py

@@ -122,6 +122,8 @@ class Atlas:
             phonemes=phonemes,
             has_tone=has_tone,
             rel_names=RELATIONS,
+            freq_zipf=arrays.get("freq_zipf"),
+            freq_rank=arrays.get("freq_rank"),
         )
         self.loaded = True
         self.load_ms = (time.perf_counter() - t0) * 1000
@@ -608,6 +610,12 @@ def walk_tree(atlas: Atlas, query: TreeQuery, root: int, t0: float) -> dict:
         node["longitude"] = info.get("longitude")
         node["glottocode"] = info.get("glottocode")
         node["iso_639_3"] = info.get("iso_639_3")
+        if atlas.ctx and atlas.ctx.freq_zipf is not None:
+            nid = int(node["id"])
+            z = float(atlas.ctx.freq_zipf[nid])
+            r = int(atlas.ctx.freq_rank[nid]) if atlas.ctx.freq_rank is not None else 0
+            node["zipf"] = round(z, 3) if z > 0 else None
+            node["freq_rank"] = r if r > 0 else None
 
     elapsed = (time.perf_counter() - t0) * 1000
     root_info = extra.get(root) or atlas.hydrate([root]).get(root, {})

backend/filters.py

@@ -30,10 +30,22 @@ class FilterContext:
     phonemes: Mapping[int, set[str]]
     has_tone: Mapping[int, bool]
     rel_names: list[str]
+    freq_zipf: object | None = None  # float32[n_nodes]
+    freq_rank: object | None = None  # uint32[n_nodes]
 
     def lang_key(self, node_id: int) -> str:
         return self.lang_vocab[int(self.lang_code[node_id])]
 
+    def zipf(self, node_id: int) -> float:
+        if self.freq_zipf is None:
+            return 0.0
+        return float(self.freq_zipf[node_id])
+
+    def rank(self, node_id: int) -> int:
+        if self.freq_rank is None:
+            return 0
+        return int(self.freq_rank[node_id])
+
     def family_name(self, node_id: int) -> str:
         lc = int(self.lang_code[node_id])
         fc = int(self.lang_family[lc])
@@ -76,6 +88,8 @@ def predicate_empty(pred: NodePredicate) -> bool:
             pred.phonemes_have,
             pred.phonemes_lack,
             pred.require_tone is not None,
+            pred.min_zipf is not None,
+            pred.max_rank is not None,
         ]
     )
 
@@ -173,6 +187,19 @@ def node_ok(node_id: int, pred: NodePredicate, ctx: FilterContext, compiled_rege
                     tone = bool(ctx.has_tone.get(gcode, False))
                     if tone is not pred.require_tone:
                         return False
+
+    if pred.min_zipf is not None or pred.max_rank is not None:
+        z = ctx.zipf(node_id)
+        r = ctx.rank(node_id)
+        unknown_freq = r <= 0 and z <= 0
+        if unknown_freq:
+            if not pred.keep_unknown:
+                return False
+        else:
+            if pred.min_zipf is not None and z < pred.min_zipf:
+                return False
+            if pred.max_rank is not None and (r <= 0 or r > pred.max_rank):
+                return False
     return True
 
 

backend/frequency.py

@@ -0,0 +1,339 @@
+from __future__ import annotations
+
+"""Modern-language word popularity (Zipf + within-language rank).
+
+Primary: Robyn Speer's ``wordfreq`` (Wikipedia, subtitles, news, books, web, social).
+Fallback: HermitDave FrequencyWords (OpenSubtitles) for languages wordfreq lacks —
+practical stand-in for popular usage when FastText crawl ``.bin`` models (multi-GB
+each) cannot ship in a Space image. Optional ``FASTTEXT_FREQ_DIR/{lang}.txt`` ranked
+vocab files are also consulted when present (one word per line, most frequent first).
+"""
+
+import math
+import os
+import re
+from functools import lru_cache
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+FREQ_DIR = ROOT / "data" / "freq"
+FASTTEXT_DIR = Path(os.environ.get("FASTTEXT_FREQ_DIR", str(FREQ_DIR / "fasttext")))
+
+# Atlas lang key / ISO 639-3 → wordfreq / FrequencyWords codes.
+LANG_TO_CODE: dict[str, str] = {
+    "english": "en",
+    "spanish": "es",
+    "french": "fr",
+    "german": "de",
+    "italian": "it",
+    "portuguese": "pt",
+    "dutch": "nl",
+    "russian": "ru",
+    "polish": "pl",
+    "swedish": "sv",
+    "norwegian": "nb",
+    "danish": "da",
+    "finnish": "fi",
+    "hungarian": "hu",
+    "czech": "cs",
+    "slovak": "sk",
+    "romanian": "ro",
+    "bulgarian": "bg",
+    "greek": "el",
+    "turkish": "tr",
+    "arabic": "ar",
+    "hebrew": "he",
+    "hindi": "hi",
+    "bengali": "bn",
+    "indonesian": "id",
+    "malay": "ms",
+    "vietnamese": "vi",
+    "thai": "th",
+    "chinese": "zh",
+    "japanese": "ja",
+    "korean": "ko",
+    "ukrainian": "uk",
+    "catalan": "ca",
+    "croatian": "hr",
+    "serbian": "sr",
+    "slovenian": "sl",
+    "lithuanian": "lt",
+    "latvian": "lv",
+    "estonian": "et",
+    "persian": "fa",
+    "urdu": "ur",
+    "tamil": "ta",
+    "tagalog": "tl",
+    "filipino": "tl",
+    "icelandic": "is",
+    "basque": "eu",
+    "galician": "gl",
+    "eng": "en",
+    "spa": "es",
+    "fra": "fr",
+    "fre": "fr",
+    "deu": "de",
+    "ger": "de",
+    "ita": "it",
+    "por": "pt",
+    "nld": "nl",
+    "dut": "nl",
+    "rus": "ru",
+    "pol": "pl",
+    "swe": "sv",
+    "nor": "nb",
+    "nob": "nb",
+    "dan": "da",
+    "fin": "fi",
+    "hun": "hu",
+    "ces": "cs",
+    "cze": "cs",
+    "slk": "sk",
+    "ron": "ro",
+    "rum": "ro",
+    "bul": "bg",
+    "ell": "el",
+    "gre": "el",
+    "tur": "tr",
+    "arb": "ar",
+    "heb": "he",
+    "hin": "hi",
+    "ben": "bn",
+    "ind": "id",
+    "msa": "ms",
+    "vie": "vi",
+    "tha": "th",
+    "cmn": "zh",
+    "zho": "zh",
+    "jpn": "ja",
+    "kor": "ko",
+    "ukr": "uk",
+    "cat": "ca",
+    "hrv": "hr",
+    "srp": "sr",
+    "slv": "sl",
+    "lit": "lt",
+    "lav": "lv",
+    "est": "et",
+    "fas": "fa",
+    "pes": "fa",
+    "urd": "ur",
+    "tam": "ta",
+    "tgl": "tl",
+    "fil": "tl",
+    "isl": "is",
+    "eus": "eu",
+    "glg": "gl",
+}
+
+# FrequencyWords filename stem on HermitDave/FrequencyWords content/2018/
+FW_FILES: dict[str, str] = {
+    "en": "en_50k.txt",
+    "es": "es_50k.txt",
+    "fr": "fr_50k.txt",
+    "de": "de_50k.txt",
+    "it": "it_50k.txt",
+    "pt": "pt_50k.txt",
+    "nl": "nl_50k.txt",
+    "ru": "ru_50k.txt",
+    "pl": "pl_50k.txt",
+    "sv": "sv_50k.txt",
+    "cs": "cs_50k.txt",
+    "ro": "ro_50k.txt",
+    "hu": "hu_50k.txt",
+    "tr": "tr_50k.txt",
+    "uk": "uk_50k.txt",
+    "fi": "fi_50k.txt",
+    "da": "da_50k.txt",
+    "el": "el_50k.txt",
+    "bg": "bg_50k.txt",
+    "hr": "hr_50k.txt",
+    "sk": "sk_50k.txt",
+    "nb": "no_50k.txt",
+    "ca": "ca_50k.txt",
+    "id": "id_50k.txt",
+    "vi": "vi_50k.txt",
+    "ar": "ar_50k.txt",
+    "he": "he_50k.txt",
+    "hi": "hi_50k.txt",
+    "ko": "ko_50k.txt",
+    "ja": "ja_50k.txt",
+    "zh": "zh_50k.txt",
+    "fa": "fa_50k.txt",
+    "tl": "tl_50k.txt",
+}
+
+FW_BASE = "https://raw.githubusercontent.com/hermitdave/FrequencyWords/master/content/2018/"
+
+_WORD_RE = re.compile(r"[^\W\d_]+", re.UNICODE)
+
+
+def freq_code_for(lang: str, iso_639_3: str | None = None) -> str | None:
+    key = (lang or "").strip().casefold()
+    if key in LANG_TO_CODE:
+        return LANG_TO_CODE[key]
+    iso = (iso_639_3 or "").strip().casefold()
+    if iso in LANG_TO_CODE:
+        return LANG_TO_CODE[iso]
+    # wordfreq uses 2-letter codes mostly
+    if len(key) == 2 and key.isalpha():
+        return key
+    return None
+
+
+def _norm_term(term: str) -> str:
+    t = (term or "").strip().casefold()
+    if t.startswith("*"):
+        t = t[1:]
+    return t
+
+
+@lru_cache(maxsize=64)
+def _wordfreq_ranks(code: str) -> dict[str, tuple[float, int]] | None:
+    try:
+        from wordfreq import get_frequency_dict, zipf_frequency, available_languages
+    except ImportError:
+        return None
+    langs = available_languages(wordlist="best")
+    if code not in langs:
+        # try large
+        langs = available_languages(wordlist="large")
+        wordlist = "large" if code in langs else None
+        if wordlist is None:
+            return None
+    else:
+        wordlist = "best"
+    freq_dict = get_frequency_dict(code, wordlist=wordlist)
+    ranked = sorted(freq_dict.items(), key=lambda kv: -kv[1])
+    out: dict[str, tuple[float, int]] = {}
+    for i, (w, _f) in enumerate(ranked, start=1):
+        # zipf_frequency is authoritative on the Zipf scale
+        z = float(zipf_frequency(w, code, wordlist=wordlist))
+        out[w] = (z, i)
+    return out
+
+
+@lru_cache(maxsize=64)
+def _ranked_file_map(path: str) -> dict[str, tuple[float, int]]:
+    """Ranked word list → (pseudo-zipf, rank). Zipf ≈ 7.5 - log10(rank)."""
+    out: dict[str, tuple[float, int]] = {}
+    p = Path(path)
+    if not p.exists():
+        return out
+    with p.open(encoding="utf-8", errors="ignore") as fh:
+        rank = 0
+        for line in fh:
+            line = line.strip()
+            if not line:
+                continue
+            # FrequencyWords: "word count" ; fasttext vocab dump: "word" or "word freq"
+            parts = line.split()
+            if not parts:
+                continue
+            w = parts[0].casefold()
+            if not w or w.startswith("#"):
+                continue
+            rank += 1
+            zipf = max(0.0, 7.5 - math.log10(rank))
+            out.setdefault(w, (zipf, rank))
+            if rank >= 1_000_000:
+                break
+    return out
+
+
+def ensure_frequencywords(code: str) -> Path | None:
+    fname = FW_FILES.get(code)
+    if not fname:
+        return None
+    FREQ_DIR.mkdir(parents=True, exist_ok=True)
+    dest = FREQ_DIR / fname
+    if dest.exists() and dest.stat().st_size > 0:
+        return dest
+    url = FW_BASE + fname
+    try:
+        import httpx
+
+        with httpx.Client(timeout=60.0, follow_redirects=True) as client:
+            r = client.get(url)
+            r.raise_for_status()
+            dest.write_bytes(r.content)
+        return dest
+    except Exception:
+        return None
+
+
+def _fasttext_path(code: str) -> Path | None:
+    for name in (f"{code}.txt", f"{code}_freq.txt", f"cc.{code}.txt"):
+        p = FASTTEXT_DIR / name
+        if p.exists():
+            return p
+    return None
+
+
+def lookup_frequency(term: str, lang: str, iso_639_3: str | None = None) -> dict:
+    """Return zipf, rank (1=most frequent), and source. Unknown → zipf 0, rank 0."""
+    code = freq_code_for(lang, iso_639_3)
+    norm = _norm_term(term)
+    empty = {"zipf": 0.0, "rank": 0, "source": None, "code": code}
+    if not code or not norm:
+        return empty
+
+    # 1) wordfreq
+    wf = _wordfreq_ranks(code)
+    if wf is not None:
+        hit = wf.get(norm)
+        if hit is None:
+            # try simple token
+            m = _WORD_RE.search(norm)
+            if m:
+                hit = wf.get(m.group(0))
+        if hit is not None:
+            return {"zipf": hit[0], "rank": hit[1], "source": "wordfreq", "code": code}
+
+    # 2) optional FastText ranked vocab dir
+    ft = _fasttext_path(code)
+    if ft is not None:
+        mp = _ranked_file_map(str(ft))
+        hit = mp.get(norm)
+        if hit is not None:
+            return {"zipf": hit[0], "rank": hit[1], "source": "fasttext", "code": code}
+
+    # 3) FrequencyWords (OpenSubtitles) popular-usage fallback
+    fw = ensure_frequencywords(code)
+    if fw is not None:
+        mp = _ranked_file_map(str(fw))
+        hit = mp.get(norm)
+        if hit is not None:
+            return {"zipf": hit[0], "rank": hit[1], "source": "frequencywords", "code": code}
+
+    return empty
+
+
+def annotate_nodes(
+    terms: list[str],
+    langs: list[str],
+    iso_by_lang: dict[str, str | None],
+) -> tuple["object", "object"]:
+    """Build per-node zipf (float32) and rank (uint32) arrays."""
+    import numpy as np
+
+    n = len(terms)
+    zipf = np.zeros(n, dtype=np.float32)
+    rank = np.zeros(n, dtype=np.uint32)
+    # Group by frequency code to reuse cached maps
+    by_code: dict[str, list[int]] = {}
+    for i, (t, lang) in enumerate(zip(terms, langs)):
+        code = freq_code_for(lang, iso_by_lang.get(lang))
+        if not code:
+            continue
+        by_code.setdefault(code, []).append(i)
+
+    for code, idxs in by_code.items():
+        # Warm caches once per code
+        sample_lang = langs[idxs[0]]
+        sample_iso = iso_by_lang.get(sample_lang)
+        for i in idxs:
+            info = lookup_frequency(terms[i], langs[i], iso_by_lang.get(langs[i]) or sample_iso)
+            zipf[i] = info["zipf"]
+            rank[i] = info["rank"]
+    return zipf, rank

backend/mmap_store.py

@@ -25,6 +25,8 @@ MMAP_FILES = {
     "lang_glotto": ("lang_glotto.u16", np.uint16),
     "lang_lat": ("lang_lat.f32", np.float32),
     "lang_lon": ("lang_lon.f32", np.float32),
+    "freq_zipf": ("freq_zipf.f32", np.float32),
+    "freq_rank": ("freq_rank.u32", np.uint32),
 }
 
 
@@ -64,7 +66,17 @@ def load_graph(index_dir: Path, copy: bool = False) -> dict[str, np.ndarray]:
     """Load mmap arrays. If copy=True, materialize into RAM (tests / tiny graphs)."""
     d = mmap_dir(index_dir)
     out: dict[str, np.ndarray] = {}
+    n_nodes = None
     for key, (fname, dtype) in MMAP_FILES.items():
-        arr = open_array(d / fname, dtype)
+        path = d / fname
+        if not path.exists():
+            # Backward compatible: older indexes without frequency arrays.
+            if key in ("freq_zipf", "freq_rank") and n_nodes is not None:
+                out[key] = np.zeros(n_nodes, dtype=dtype)
+                continue
+            raise FileNotFoundError(path)
+        arr = open_array(path, dtype)
         out[key] = np.array(arr) if copy else arr
+        if key == "lang_code":
+            n_nodes = len(out[key])
     return out

backend/models.py

@@ -32,6 +32,10 @@ class NodePredicate(BaseModel):
     phonemes_have: list[str] = Field(default_factory=list)
     phonemes_lack: list[str] = Field(default_factory=list)
     require_tone: bool | None = None
+    # Popularity within language (modern lects). Zipf ~3 rare … ~7 very common.
+    min_zipf: float | None = Field(default=None, ge=0.0, le=8.0)
+    # Keep leaves whose corpus rank is <= N (1 = most frequent). Orders of magnitude presets in UI.
+    max_rank: int | None = Field(default=None, ge=1, le=5_000_000)
     keep_unknown: bool = False
 
 

frontend/src/App.tsx

@@ -70,11 +70,13 @@ function PredicateEditor({
   value,
   onChange,
   extra,
+  showPopularity = false,
 }: {
   meta: Meta;
   value: NodePredicate;
   onChange: (p: NodePredicate) => void;
   extra?: ReactNode;
+  showPopularity?: boolean;
 }) {
   const langOpts = useMemo(
     () => meta.languages.slice(0, 2500).map((l) => ({ key: l.key, label: `${l.display || l.key} (${l.nodes})` })),
@@ -132,6 +134,43 @@ function PredicateEditor({
         <option value="yes">has tone</option>
         <option value="no">no tone</option>
       </select>
+      {showPopularity && (
+        <>
+          <label className="field">Popularity in language</label>
+          <p style={{ color: "var(--muted)", fontSize: 12, margin: "0 0 6px" }}>
+            Modern lects via wordfreq (Zipf); fallback ranked lists for extra languages. Historical forms usually have no score.
+          </p>
+          <label className="field">Top N most frequent</label>
+          <select
+            value={value.max_rank ?? ""}
+            onChange={(e) => onChange({ ...value, max_rank: e.target.value === "" ? null : Number(e.target.value) })}
+          >
+            <option value="">any</option>
+            <option value={100}>top 100</option>
+            <option value={1000}>top 1,000</option>
+            <option value={10000}>top 10,000</option>
+            <option value={100000}>top 100,000</option>
+            <option value={1000000}>top 1,000,000</option>
+          </select>
+          <label className="field">
+            Min Zipf {value.min_zipf != null ? value.min_zipf.toFixed(1) : "off"}
+          </label>
+          <input
+            type="range"
+            min={0}
+            max={70}
+            step={1}
+            value={value.min_zipf == null ? 0 : Math.round(value.min_zipf * 10)}
+            onChange={(e) => {
+              const raw = Number(e.target.value);
+              onChange({ ...value, min_zipf: raw <= 0 ? null : raw / 10 });
+            }}
+          />
+          <p style={{ color: "var(--muted)", fontSize: 12, margin: "4px 0 0" }}>
+            Zipf ≈ 3 rare · 4 uncommon · 5 everyday · 6+ very common. 0 = no minimum.
+          </p>
+        </>
+      )}
       <label className="field">WALS feature</label>
       <select
         value={value.wals[0]?.feature_id || ""}
@@ -415,6 +454,8 @@ export default function App() {
   const chipCount =
     leaf.languages.length +
     leaf.families.length +
+    (leaf.max_rank ? 1 : 0) +
+    (leaf.min_zipf != null ? 1 : 0) +
     pathNode.languages.length +
     pathNode.families.length +
     pathRelAny.length +
@@ -565,7 +606,7 @@ export default function App() {
                   <p style={{ color: "var(--muted)", fontSize: 13 }}>
                     Matching words in the filtered language stay visible even when they have further descendants.
                   </p>
-                  <PredicateEditor meta={meta} value={leaf} onChange={setLeaf} />
+                  <PredicateEditor meta={meta} value={leaf} onChange={setLeaf} showPopularity />
                 </div>
               )}
               {filterTab === "path" && (
@@ -753,6 +794,10 @@ export default function App() {
                 <dd>{selected.macroarea || "—"}</dd>
                 <dt>Glottocode</dt>
                 <dd>{selected.glottocode || "—"}</dd>
+                <dt>Zipf</dt>
+                <dd>{selected.zipf != null ? selected.zipf : "—"}</dd>
+                <dt>Freq rank</dt>
+                <dd>{selected.freq_rank != null ? selected.freq_rank.toLocaleString() : "—"}</dd>
               </dl>
               {selected.kind === "word" && (
                 <div className="row">

frontend/src/types.ts

@@ -19,6 +19,8 @@ export interface NodePredicate {
   phonemes_have: string[];
   phonemes_lack: string[];
   require_tone: boolean | null;
+  min_zipf?: number | null;
+  max_rank?: number | null;
   keep_unknown: boolean;
 }
 
@@ -65,6 +67,8 @@ export interface GraphNode {
   longitude?: number | null;
   glottocode?: string | null;
   iso_639_3?: string | null;
+  zipf?: number | null;
+  freq_rank?: number | null;
   count?: number;
   expand_key?: string;
   relation_mix?: Record<string, number>;
@@ -148,6 +152,8 @@ export function emptyPredicate(): NodePredicate {
     phonemes_have: [],
     phonemes_lack: [],
     require_tone: null,
+    min_zipf: null,
+    max_rank: null,
     keep_unknown: false,
   };
 }

pipeline/build_index.py

@@ -375,6 +375,14 @@ def main() -> None:
         raw = np.diff(rev_off)
         child_count = np.minimum(raw, 65535).astype(np.uint16)
 
+    print("word frequencies (wordfreq + popular-usage fallback)")
+    from backend.frequency import annotate_nodes
+
+    iso_by_lang = {r["lang_key"]: r.get("iso_639_3") for r in lang_rows}
+    freq_zipf, freq_rank = annotate_nodes(terms, langs, iso_by_lang)
+    known = int(np.count_nonzero(freq_rank))
+    print(f"  frequency known for {known:,}/{n_nodes:,} nodes")
+
     print("WALS / phonemes / cognates")
     wals_langs = pd.read_csv(RAW / "wals_languages.csv", low_memory=False)
     wals_codes = pd.read_csv(RAW / "wals_codes.csv")
@@ -579,6 +587,8 @@ def main() -> None:
             "lang_glotto": lang_glotto,
             "lang_lat": lang_lat,
             "lang_lon": lang_lon,
+            "freq_zipf": freq_zipf,
+            "freq_rank": freq_rank,
         },
     )
 

requirements.txt

@@ -1,3 +1,4 @@
+wordfreq>=3.1.0
 fastapi>=0.115.0
 uvicorn[standard]>=0.32.0
 numpy>=2.0.0