Spaces:
Paused
Paused
File size: 1,612 Bytes
20857b0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 | from dataclasses import dataclass, field
@dataclass
class SemanticTokenEntry:
token_id: int
label: str = ""
category: str = ""
embedding: list[float] | None = None
frequency: int = 0
confidence: float = 0.0
class SemanticDictionary:
"""Maps token IDs to human-readable semantic meaning.
Rather than treating tokens as anonymous codebook entries, each
token carries semantic metadata — enabling search, editing, and
reasoning directly in token space without neural decode.
Schema:
TokenEntry {
TokenID int
Label str "Red Sports Car"
Category str "vehicle.car.sports"
Embedding float[] semantic vector
Frequency int occurrence count
Confidence float 0-1
}
"""
def __init__(self):
self._entries: dict[int, SemanticTokenEntry] = {}
def register(self, entry: SemanticTokenEntry):
self._entries[entry.token_id] = entry
def lookup(self, token_id: int) -> SemanticTokenEntry | None:
return self._entries.get(token_id)
def search(self, query: str) -> list[SemanticTokenEntry]:
q = query.lower()
return [e for e in self._entries.values() if q in e.label.lower()]
def search_by_category(self, category: str) -> list[SemanticTokenEntry]:
return [e for e in self._entries.values() if e.category.startswith(category)]
@property
def size(self) -> int:
return len(self._entries)
def clear(self):
self._entries.clear()
|