Add HDC text-to-pictogram space

README.md

@@ -1,12 +1,22 @@
 ---
-title: HDC ToPicto Translation
-emoji: 👀
-colorFrom: indigo
-colorTo: purple
+title: Speech to ARASAAC Pictograms (HDC)
+emoji: 🧠
+colorFrom: purple
+colorTo: blue
 sdk: gradio
-sdk_version: 6.9.0
 app_file: app.py
 pinned: false
+license: apache-2.0
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# 🧠 Speech / Text → ARASAAC Pictograms (HDC)
+
+Convert spoken or written English into ARASAAC pictograms using **Hyperdimensional Computing**.
+
+- **Audio tab**: record via microphone or upload a `.wav` file → transcribed with [Whisper tiny (EN)](https://huggingface.co/openai/whisper-tiny.en) → pictograms
+- **Text tab**: type directly → pictograms
+- Transcription is editable before generating pictograms
+- Pictogram lookup uses an **HDC prototype memory** built from ~855 core vocabulary pictograms
+  with WordNet synonym injection — no API call per word at inference time
+- Each pictogram card shows a **similarity score badge** (retrieval confidence)
+- Words below the confidence threshold show a `?` placeholder

app.py

@@ -0,0 +1,235 @@
+import re
+import json
+import os
+import nltk
+import gradio as gr
+from transformers import pipeline
+from nltk.corpus import wordnet
+
+# Ensure WordNet data is available
+nltk.download("wordnet", quiet=True)
+
+# ── HDC imports ───────────────────────────────────────────────────────────────
+from hdc_text2picto import encode_word, PictogramMemory
+
+# ── ASR model ─────────────────────────────────────────────────────────────────
+asr = pipeline(
+    "automatic-speech-recognition",
+    model="openai/whisper-tiny.en",
+    device="cpu",
+)
+
+# ── HDC: build prototype memory from cached core pictograms ───────────────────
+#
+# At startup we load the locally cached ARASAAC core vocabulary JSON and build
+# a PictogramMemory by encoding every keyword (+ WordNet synonyms) for each
+# pictogram. This replaces the per-word ARASAAC API call at inference time:
+# retrieval is entirely local and offline after startup.
+
+SYNSET_SUFFIX_TO_WN = {
+    "n": wordnet.NOUN, "v": wordnet.VERB,
+    "a": wordnet.ADJ,  "s": wordnet.ADJ, "r": wordnet.ADV,
+}
+SYNSET_SUFFIX_TO_POS = {"n": "NOUN", "v": "VERB", "a": "ADJ", "s": "ADJ", "r": "ADV"}
+
+CONFIDENCE_THRESHOLD = 0.10   # cosine similarity below this → show ? placeholder
+
+
+def pos_from_synsets(synsets: list[str]) -> str:
+    if synsets:
+        return SYNSET_SUFFIX_TO_POS.get(synsets[0].split("-")[-1], "OTHER")
+    return "OTHER"
+
+
+def get_synonyms(keyword: str, synsets: list[str]) -> list[str]:
+    wn_pos = SYNSET_SUFFIX_TO_WN.get(synsets[0].split("-")[-1]) if synsets else None
+    synonyms = set()
+    for ss in wordnet.synsets(keyword, pos=wn_pos):
+        for lemma in ss.lemmas():
+            syn = lemma.name().replace("_", " ").lower()
+            if syn != keyword.lower():
+                synonyms.add(syn)
+    return list(synonyms)
+
+
+def build_memory(core_pictos: list[dict]) -> PictogramMemory:
+    """Encode all core pictogram keywords (+ WordNet synonyms) into prototypes."""
+    memory = PictogramMemory()
+    for p in core_pictos:
+        pid      = p["_id"]
+        synsets  = p.get("synsets", [])
+        keywords = [kw for kw in p.get("keywords", []) if kw.get("keyword")]
+        if not keywords:
+            continue
+
+        label = keywords[0]["keyword"]
+        pos   = pos_from_synsets(synsets)
+
+        # Encode using pos="OTHER" and synsets=[] to match inference-time encoding,
+        # where POS and synsets are unknown. This ensures training and inference
+        # composites are built the same way, so cosine similarity is meaningful.
+        seen = set()
+        for kw in keywords:
+            word = kw["keyword"]
+            if word.lower() not in seen:
+                seen.add(word.lower())
+                memory.add(pid, encode_word(word, "OTHER", "NONE", []), label)
+
+            # WordNet synonym injection (encoded the same way)
+            for syn in get_synonyms(word, synsets):
+                if syn not in seen:
+                    seen.add(syn)
+                    memory.add(pid, encode_word(syn, "OTHER", "NONE", []), label)
+
+    memory.build()
+    return memory
+
+
+print("Building HDC prototype memory from core vocabulary...")
+_cache_path = os.path.join(os.path.dirname(__file__), "core_pictograms.json")
+with open(_cache_path) as f:
+    _core_pictos = json.load(f)
+memory = build_memory(_core_pictos)
+print(f"  Ready — {len(memory.protos)} pictogram prototypes loaded.")
+
+# ── HDC lookup ────────────────────────────────────────────────────────────────
+
+def hdc_lookup(word: str) -> tuple[int | None, float, str]:
+    """
+    Encode a word as an HDC composite vector and retrieve the nearest pictogram
+    prototype. POS and synsets are unknown at inference time so we use defaults;
+    the semantic content from the GloVe embedding carries most of the signal.
+
+    Returns (picto_id, similarity, label) or (None, 0.0, "") if below threshold.
+    """
+    query_hv = encode_word(word, pos="OTHER", ner="NONE", synsets=[])
+    results  = memory.retrieve(query_hv, top_k=1)
+    pid, label, sim = results[0]
+    if sim >= CONFIDENCE_THRESHOLD:
+        return pid, sim, label
+    return None, sim, ""
+
+# ── Image URL ─────────────────────────────────────────────────────────────────
+
+def picto_url(picto_id: int, size: int = 500) -> str:
+    return f"https://static.arasaac.org/pictograms/{picto_id}/{picto_id}_{size}.png"
+
+# ── Tokeniser ─────────────────────────────────────────────────────────────────
+
+def tokenize(text: str) -> list[str]:
+    return [re.sub(r"[^\w'-]", "", tok) for tok in text.split() if re.sub(r"[^\w'-]", "", tok)]
+
+# ── Render pictograms ─────────────────────────────────────────────────────────
+
+def render_pictos(text: str) -> str:
+    if not text or not text.strip():
+        return "<p style='color:gray;text-align:center;padding:20px;'>No text to display.</p>"
+
+    cards = []
+    for word in tokenize(text):
+        picto_id, sim, label = hdc_lookup(word)
+
+        if picto_id:
+            img = (
+                f'<img src="{picto_url(picto_id)}" alt="{word}" title="{label} (sim={sim:.2f})" '
+                f'style="width:110px;height:110px;object-fit:contain;">'
+            )
+            # Similarity badge: green if confident, orange if marginal
+            badge_color = "#4caf50" if sim >= 0.15 else "#ff9800"
+            badge = (
+                f'<span style="font-size:0.7rem;background:{badge_color};color:white;'
+                f'border-radius:4px;padding:1px 4px;">{sim:.2f}</span>'
+            )
+            label_style = "font-size:0.85rem;margin-top:4px;word-break:break-word;font-weight:600;"
+            label_html  = f'<p style="{label_style}">{word}</p>{badge}'
+        else:
+            img = (
+                '<div style="width:110px;height:110px;background:#f0f0f0;border-radius:8px;'
+                'display:flex;align-items:center;justify-content:center;font-size:2rem;color:#bbb;">?</div>'
+            )
+            label_style = "font-size:0.85rem;margin-top:4px;word-break:break-word;color:#aaa;"
+            label_html  = f'<p style="{label_style}">{word}</p>'
+
+        cards.append(
+            f'<div style="display:flex;flex-direction:column;align-items:center;width:130px;'
+            f'padding:8px;background:white;border-radius:10px;box-shadow:0 1px 4px rgba(0,0,0,0.1);">'
+            f'{img}{label_html}</div>'
+        )
+
+    return (
+        '<div style="display:flex;flex-wrap:wrap;gap:12px;justify-content:center;'
+        'padding:20px;background:#f5f5f5;border-radius:12px;">'
+        + "".join(cards) + "</div>"
+    )
+
+# ── Processing functions ──────────────────────────────────────────────────────
+
+def process_audio(audio_path):
+    if audio_path is None:
+        return "", "<p style='color:gray;text-align:center;padding:20px;'>No audio provided.</p>"
+    result = asr(audio_path)
+    text   = result["text"].strip()
+    return text, render_pictos(text)
+
+
+def process_text(text):
+    return render_pictos(text)
+
+# ── Gradio UI ─────────────────────────────────────────────────────────────────
+
+with gr.Blocks(title="Speech/Text → ARASAAC Pictograms (HDC)") as demo:
+    gr.Markdown(
+        """
+        # 🧠 Speech / Text → ARASAAC Pictograms (HDC)
+        Convert spoken or written English into ARASAAC pictograms using
+        **Hyperdimensional Computing** for offline, semantic word-to-pictogram retrieval.
+
+        Uses [Whisper tiny](https://huggingface.co/openai/whisper-tiny.en) for speech recognition.
+        Pictogram lookup uses HDC prototype memory built from ~855 core vocabulary pictograms
+        and WordNet synonym injection — no API call per word at inference time.
+
+        The similarity score badge on each card shows retrieval confidence
+        (🟢 ≥ 0.15 · 🟠 < 0.15 · **?** below threshold).
+        """
+    )
+
+    with gr.Tab("🎤 Audio"):
+        audio_input = gr.Audio(
+            sources=["microphone", "upload"],
+            type="filepath",
+            label="Record or upload audio (.wav)",
+        )
+        transcribe_btn  = gr.Button("Transcribe & Generate Pictograms", variant="primary")
+        transcribed_box = gr.Textbox(
+            label="Transcribed text (editable — press Enter to regenerate pictograms)",
+            lines=2,
+            interactive=True,
+        )
+        audio_picto_output = gr.HTML()
+
+        transcribe_btn.click(
+            fn=process_audio,
+            inputs=audio_input,
+            outputs=[transcribed_box, audio_picto_output],
+        )
+        transcribed_box.submit(
+            fn=process_text,
+            inputs=transcribed_box,
+            outputs=audio_picto_output,
+        )
+
+    with gr.Tab("✍️ Text"):
+        text_input = gr.Textbox(
+            label="Input text",
+            placeholder="e.g. I want to eat an apple",
+            lines=2,
+        )
+        text_btn        = gr.Button("Generate Pictograms", variant="primary")
+        text_picto_output = gr.HTML()
+
+        text_btn.click(fn=process_text, inputs=text_input, outputs=text_picto_output)
+        text_input.submit(fn=process_text, inputs=text_input, outputs=text_picto_output)
+
+
+if __name__ == "__main__":
+    demo.launch(theme=gr.themes.Soft())

hdc_text2picto.py

@@ -0,0 +1,263 @@
+"""
+Minimal HDC Text-to-Pictogram example.
+
+Encoding notation:
+    word_hv   = project(embedding(word))
+    composite = bundle(
+        bind(word_hv, hv(NER_class)),
+        bind(word_hv, hv(POS_tag)),
+        bind(word_hv, hv(WN_synset)),
+        word_hv
+    )
+"""
+
+import numpy as np
+import requests
+from sentence_transformers import SentenceTransformer
+
+# ---------------------------------------------------------------------------
+# Hyperparameters
+# ---------------------------------------------------------------------------
+D = 10_000          # hypervector dimension
+EMBEDDING_MODEL = "sentence-transformers/average_word_embeddings_glove.6B.300d"  # static word vectors, 300-dim
+EMBEDDING_DIM   = 300
+ARASAAC_API = "https://api.arasaac.org/v1/pictograms/en"
+RNG = np.random.default_rng(42)
+
+# ---------------------------------------------------------------------------
+# 1. Atom memory — fixed random bipolar hypervectors for symbolic features
+#
+# In HDC, atomic concepts (POS tags, NER classes, synset IDs) are each
+# represented by a unique, randomly generated hypervector. These are called
+# "atom vectors". Because D is very large (~10,000), any two randomly drawn
+# vectors are nearly orthogonal with high probability — meaning they are
+# maximally dissimilar by default. This quasi-orthogonality is the key
+# property that lets us encode distinct features without interference.
+#
+# Atom vectors are fixed for the lifetime of the model: the same symbol
+# always maps to the same vector, so representations are consistent across
+# all words and queries.
+# ---------------------------------------------------------------------------
+POS_HVS = {
+    # One atom per part-of-speech tag. Binding word_hv with e.g. POS_HVS["VERB"]
+    # produces a new vector that encodes "this word used as a verb" — distinct
+    # from the same word bound to POS_HVS["NOUN"].
+    tag: RNG.choice([-1.0, 1.0], size=D)
+    for tag in ["NOUN", "VERB", "ADJ", "ADV", "PROPN", "OTHER"]
+}
+
+NER_HVS = {
+    # One atom per named-entity class. Allows the model to distinguish e.g.
+    # "Jordan" as a PERSON vs "Jordan" as a LOC, which may map to different
+    # pictograms.
+    cls: RNG.choice([-1.0, 1.0], size=D)
+    for cls in ["PERSON", "ORG", "LOC", "DATE", "NONE"]
+}
+
+# WordNet synset atoms are created on demand and cached.
+# Synsets provide a language-neutral semantic identifier (e.g. "01170802-v"
+# for the concept of eating). Using synset atoms ties the word representation
+# to an abstract meaning rather than a surface form, enabling cross-lingual
+# matching: a French word and its English equivalent share the same synset
+# atom if they are linked in a multilingual WordNet.
+_SYNSET_HVS: dict[str, np.ndarray] = {}
+
+def hv(synset_id: str) -> np.ndarray:
+    """Return the (cached) atom hypervector for a WordNet synset ID."""
+    if synset_id not in _SYNSET_HVS:
+        _SYNSET_HVS[synset_id] = RNG.choice([-1.0, 1.0], size=D)
+    return _SYNSET_HVS[synset_id]
+
+# ---------------------------------------------------------------------------
+# 2. Random projection matrix: embedding_dim → D
+#
+# FastText produces a 300-dim dense vector for each word. We need to lift
+# this into the HD space (10,000-dim) so that HDC operations can be applied.
+# A random Gaussian matrix W achieves this: by the Johnson-Lindenstrauss
+# lemma, random projections approximately preserve pairwise distances, so
+# words that were semantically similar in embedding space remain similar
+# after projection. Dividing by sqrt(EMBEDDING_DIM) keeps the scale stable.
+# The sign() in project() binarises the result to {-1, +1}, making it a
+# proper bipolar hypervector.
+# ---------------------------------------------------------------------------
+_model = SentenceTransformer(EMBEDDING_MODEL)
+W = RNG.standard_normal((D, EMBEDDING_DIM)) / np.sqrt(EMBEDDING_DIM)
+
+# ---------------------------------------------------------------------------
+# 3. HDC operations
+# ---------------------------------------------------------------------------
+def project(embedding: np.ndarray) -> np.ndarray:
+    """Lift a dense embedding into HD space via random projection.
+
+    sign(W @ v) maps the continuous embedding to a bipolar hypervector
+    {-1, +1}^D while approximately preserving cosine similarity.
+    """
+    return np.sign(W @ embedding)
+
+def bind(a: np.ndarray, b: np.ndarray) -> np.ndarray:
+    """Binding: element-wise multiplication of two bipolar hypervectors.
+
+    bind(a, b) produces a vector that is dissimilar to both a and b
+    individually, but encodes their *association*. It is the HDC equivalent
+    of a key-value pair: bind(word_hv, POS_HVS["VERB"]) means
+    "this word in its role as a verb". Binding is invertible:
+    bind(bind(a, b), b) == a, so features can be recovered later.
+    """
+    return a * b
+
+def bundle(vectors: list[np.ndarray]) -> np.ndarray:
+    """Bundling: element-wise majority vote across a list of hypervectors.
+
+    bundle([a, b, c]) produces a vector that is *similar* to all of its
+    inputs simultaneously. It is the HDC equivalent of a set: the result
+    represents "a and b and c together". Used here to combine the
+    different feature bindings into a single composite word vector.
+    """
+    return np.sign(np.sum(vectors, axis=0))
+
+def cosine_sim(a: np.ndarray, b: np.ndarray) -> float:
+    return float(np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b) + 1e-9))
+
+# ---------------------------------------------------------------------------
+# 4. Word encoder — follows the notation exactly
+#
+# Each word is encoded as a composite hypervector that fuses:
+#   - its semantic content (via the projected FastText embedding)
+#   - its grammatical role (POS tag atom)
+#   - its named-entity class (NER atom)
+#   - its abstract meaning (WordNet synset atom(s))
+#
+# The bind operations associate the word's semantic vector with each feature
+# atom. The final bundle merges all of these associations into one vector.
+# Two words with similar embeddings *and* the same POS/NER/synset will
+# produce very similar composites — making them likely to activate the same
+# pictogram prototype.
+# ---------------------------------------------------------------------------
+def encode_word(word: str, pos: str, ner: str, synsets: list[str]) -> np.ndarray:
+    # Multi-word expressions (e.g. "swimming costume", "gum boots") are split
+    # into tokens and each token is encoded independently then bundled together.
+    # This avoids the OOV problem where GloVe has no entry for the full phrase
+    # but knows each constituent word well.
+    tokens = word.split()
+    if len(tokens) > 1:
+        return bundle([encode_word(t, pos, ner, synsets) for t in tokens])
+
+    embedding = _model.encode(word)   # static GloVe vector; swap for fasttext.get_word_vector() when cc.en.300.bin is available
+    word_hv   = project(embedding)             # lift to HD space
+
+    components = [
+        bind(word_hv, NER_HVS.get(ner, NER_HVS["NONE"])),   # bind(word_hv, hv(NER_class))
+        bind(word_hv, POS_HVS.get(pos, POS_HVS["OTHER"])),   # bind(word_hv, hv(POS_tag))
+        *[bind(word_hv, hv(syn)) for syn in synsets],         # bind(word_hv, hv(WN_synset))
+        word_hv,                                               # base representation
+    ]
+    return bundle(components)   # composite vector for this word
+
+# ---------------------------------------------------------------------------
+# 5. Prototype memory
+#
+# A prototype is an aggregated hypervector representing all the words that
+# map to a given pictogram. During training, composite vectors for each
+# training word are summed (accumulated). After all words are processed,
+# a majority vote (sign) finalises each prototype into a bipolar vector.
+#
+# The resulting prototype sits near the "centre" of all its training words
+# in HD space — analogous to a centroid classifier in standard ML. At
+# inference, the query composite is compared to every prototype via cosine
+# similarity, and the closest one wins.
+#
+# Key advantage: adding a new pictogram (or a new word sense) requires only
+# bundling one more composite into the relevant accumulator — no retraining.
+# ---------------------------------------------------------------------------
+class PictogramMemory:
+    def __init__(self):
+        self._accum:  dict[int, np.ndarray] = {}   # running sum before finalisation
+        self.protos:  dict[int, np.ndarray] = {}   # finalised bipolar prototypes
+        self.labels:  dict[int, str]        = {}   # picto_id -> primary keyword
+
+    def add(self, picto_id: int, composite: np.ndarray, label: str = ""):
+        """Accumulate a composite vector into the prototype for picto_id."""
+        if picto_id not in self._accum:
+            self._accum[picto_id] = np.zeros(D)
+            self.labels[picto_id] = label
+        self._accum[picto_id] += composite
+
+    def build(self):
+        """Finalise all prototypes via majority vote (sign of accumulated sum)."""
+        self.protos = {pid: np.sign(acc) for pid, acc in self._accum.items()}
+
+    def retrieve(self, query: np.ndarray, top_k: int = 3) -> list[tuple]:
+        """Return top-k (picto_id, label, similarity) sorted by cosine similarity."""
+        scores = [
+            (pid, self.labels[pid], cosine_sim(query, proto))
+            for pid, proto in self.protos.items()
+        ]
+        return sorted(scores, key=lambda x: -x[2])[:top_k]
+
+# ---------------------------------------------------------------------------
+# 6. ARASAAC helpers
+# ---------------------------------------------------------------------------
+def fetch_synsets(picto_id: int) -> tuple[list[str], str]:
+    """Fetch WordNet synset IDs and primary keyword for a pictogram from the API."""
+    r = requests.get(f"{ARASAAC_API}/{picto_id}", timeout=10)
+    r.raise_for_status()
+    data    = r.json()
+    synsets = data.get("synsets", [])
+    label   = (data.get("keywords") or [{}])[0].get("keyword", str(picto_id))
+    return synsets, label
+
+# ---------------------------------------------------------------------------
+# 7. Demo
+# ---------------------------------------------------------------------------
+# Training set: (word, POS, NER, picto_id)
+# POS/NER would come from a tagger at runtime; hardcoded here for clarity.
+TRAIN = [
+    ("eat",      "VERB", "NONE", 6456),
+    ("eating",   "VERB", "NONE", 6456),
+    ("food",     "NOUN", "NONE", 6456),
+    ("drink",    "VERB", "NONE", 2276),
+    ("water",    "NOUN", "NONE", 2276),
+    ("run",      "VERB", "NONE", 2719),
+    ("running",  "VERB", "NONE", 2719),
+    ("house",    "NOUN", "NONE", 2317),
+    ("home",     "NOUN", "NONE", 2317),
+    ("happy",    "ADJ",  "NONE", 3245),
+    ("sad",      "ADJ",  "NONE", 2606),
+]
+
+# Query set: words not seen during training
+QUERIES = [
+    ("consume",  "VERB", "NONE", []),
+    ("joyful",   "ADJ",  "NONE", []),
+    ("dwelling", "NOUN", "NONE", []),
+    ("sprint",   "VERB", "NONE", []),
+    ("beverage", "NOUN", "NONE", []),
+]
+
+if __name__ == "__main__":
+    memory = PictogramMemory()
+    synset_cache: dict[int, tuple[list[str], str]] = {}
+
+    # --- Training: encode each word and accumulate into its pictogram prototype ---
+    print("=== Training ===")
+    for word, pos, ner, picto_id in TRAIN:
+        if picto_id not in synset_cache:
+            synset_cache[picto_id] = fetch_synsets(picto_id)
+        synsets, label = synset_cache[picto_id]
+
+        composite = encode_word(word, pos, ner, synsets)
+        memory.add(picto_id, composite, label)
+        print(f"  '{word}' ({pos}) + synsets={synsets} → picto {picto_id} [{label}]")
+
+    memory.build()
+    print(f"\nBuilt {len(memory.protos)} prototypes.\n")
+
+    # --- Retrieval: encode unseen words and find the nearest prototype ---
+    print("=== Retrieval ===")
+    for word, pos, ner, synsets in QUERIES:
+        query_hv = encode_word(word, pos, ner, synsets)
+        results  = memory.retrieve(query_hv, top_k=3)
+        print(f"  Query: '{word}'")
+        for pid, label, sim in results:
+            print(f"    → [{pid}] {label:<20s}  sim={sim:.4f}")
+        print()

requirements.txt

@@ -0,0 +1,8 @@
+gradio>=6.0.0
+transformers
+torch
+requests
+soundfile
+librosa
+sentence-transformers
+nltk