| """Motor de TokenScope (lado Space). |
| |
| Mode 1: pide la distribucion del siguiente token a un endpoint de Modal (Qwen). |
| Mode 2: pide la similitud semantica entre dos palabras a otro endpoint (Nemotron |
| Embed VL). Los modelos corren en Modal; el Space queda sin GPU. |
| |
| STUB local para probar sin red ni GPU: TOKENSCOPE_STUB=1 |
| """ |
| import json |
| import os |
| import re |
| import unicodedata |
| import urllib.request |
|
|
| STUB = os.environ.get("TOKENSCOPE_STUB") == "1" |
|
|
| LOGITS_URL = os.environ.get( |
| "TOKENSCOPE_LOGITS_URL", |
| "https://sanightblack--tokenscope-logits-logits-topk.modal.run", |
| ) |
| EMBED_URL = os.environ.get( |
| "TOKENSCOPE_EMBED_URL", |
| "https://sanightblack--tokenscope-embed-embed-similarity.modal.run", |
| ) |
|
|
| SEARCH_K = 200 |
| SHOW_K = 10 |
|
|
|
|
| def normalize(word: str) -> str: |
| word = word.strip().lower() |
| word = "".join( |
| c for c in unicodedata.normalize("NFD", word) |
| if unicodedata.category(c) != "Mn" |
| ) |
| return re.sub(r"[^\w]", "", word) |
|
|
|
|
| def first_word(text: str) -> str: |
| parts = text.strip().split() |
| return parts[0] if parts else "" |
|
|
|
|
| def _post(url: str, payload: dict, timeout: int = 180) -> dict: |
| data = json.dumps(payload).encode("utf-8") |
| req = urllib.request.Request( |
| url, data=data, headers={"Content-Type": "application/json"}, method="POST", |
| ) |
| with urllib.request.urlopen(req, timeout=timeout) as res: |
| return json.loads(res.read().decode("utf-8")) |
|
|
|
|
| |
| |
| |
| def _dist_stub(context: str, k: int): |
| import hashlib |
| import math |
|
|
| base = ["the", "a", "of", "and", "to", "in", "is", "with", "for", "on", |
| "it", "you", "that", "be", "as", "more", "home", "water", "sun", "eyes"] |
| seed = int(hashlib.md5(context.encode()).hexdigest(), 16) |
| rng = [(seed >> (i * 7)) % 1000 for i in range(len(base))] |
| weighted = sorted(zip(base, rng), key=lambda x: -x[1])[:k] |
| total = sum(math.exp(w / 200) for _, w in weighted) |
| return [(" " + tok, math.exp(w / 200) / total) for tok, w in weighted] |
|
|
|
|
| def next_token_dist(context: str, k: int = SHOW_K): |
| if STUB: |
| return _dist_stub(context, k) |
| return [(tok, float(p)) for tok, p in _post(LOGITS_URL, {"context": context, "k": k})["dist"]] |
|
|
|
|
| def analyze(context: str, player_word: str): |
| dist = next_token_dist(context, SEARCH_K) |
| target = normalize(first_word(player_word)) |
| rank, prob = None, 0.0 |
| if target: |
| for r, (tok, p) in enumerate(dist, start=1): |
| tn = normalize(tok) |
| if not tn: |
| continue |
| if tn == target or target.startswith(tn): |
| rank, prob = r, p |
| break |
| return dist[:SHOW_K], rank, prob |
|
|
|
|
| def score_for_rank(rank): |
| if rank is None: |
| return 0 |
| if rank == 1: |
| return 100 |
| if rank <= 5: |
| return 50 |
| if rank <= 10: |
| return 25 |
| if rank <= 50: |
| return 10 |
| return 5 |
|
|
|
|
| |
| |
| |
| def _sim_stub(guess: str, target: str) -> float: |
| a, b = normalize(guess), normalize(target) |
| if a == b: |
| return 1.0 |
| common = len(set(a) & set(b)) |
| return min(0.92, 0.15 + 0.09 * common) |
|
|
|
|
| def embed_similarity(guess: str, target: str) -> float: |
| if STUB: |
| return _sim_stub(guess, target) |
| return float(_post(EMBED_URL, {"guess": guess, "target": target})["similarity"]) |
|
|
