semantic/dictionary.py

from dataclasses import dataclass, field


@dataclass
class SemanticTokenEntry:
    token_id: int
    label: str = ""
    category: str = ""
    embedding: list[float] | None = None
    frequency: int = 0
    confidence: float = 0.0


class SemanticDictionary:
    """Maps token IDs to human-readable semantic meaning.

    Rather than treating tokens as anonymous codebook entries, each
    token carries semantic metadata — enabling search, editing, and
    reasoning directly in token space without neural decode.

    Schema:
        TokenEntry {
            TokenID       int
            Label         str         "Red Sports Car"
            Category      str         "vehicle.car.sports"
            Embedding     float[]     semantic vector
            Frequency     int         occurrence count
            Confidence    float       0-1
        }
    """
    def __init__(self):
        self._entries: dict[int, SemanticTokenEntry] = {}

    def register(self, entry: SemanticTokenEntry):
        self._entries[entry.token_id] = entry

    def lookup(self, token_id: int) -> SemanticTokenEntry | None:
        return self._entries.get(token_id)

    def search(self, query: str) -> list[SemanticTokenEntry]:
        q = query.lower()
        return [e for e in self._entries.values() if q in e.label.lower()]

    def search_by_category(self, category: str) -> list[SemanticTokenEntry]:
        return [e for e in self._entries.values() if e.category.startswith(category)]

    @property
    def size(self) -> int:
        return len(self._entries)

    def clear(self):
        self._entries.clear()