Hermes Bot commited on
feat: add user-configurable HF token + model override
Browse files- Add /api/config GET/POST endpoints for BYO HF token + model id
- Token stored in-memory only, never written to disk
- UI: config button + modal in CRT terminal footer
- Frontend: localStorage-free, all state server-side per-session
- Procedural fallback always active when inference fails or cooldown hits
- projects/tinybard/app.py +59 -6
- projects/tinybard/static/index.html +21 -1
- projects/tinybard/static/main.js +46 -0
- shared/inference_client.py +1 -1
projects/tinybard/app.py
CHANGED
|
@@ -19,7 +19,9 @@ import random
|
|
| 19 |
import logging
|
| 20 |
import sys
|
| 21 |
from pathlib import Path
|
| 22 |
-
from typing import Dict, List
|
|
|
|
|
|
|
| 23 |
|
| 24 |
import gradio as gr
|
| 25 |
from fastapi import FastAPI
|
|
@@ -57,6 +59,26 @@ STATIC_DIR = BASE_DIR / "static"
|
|
| 57 |
# Cooldown enforced in shared.inference_client.
|
| 58 |
TINYBARD_MODEL = os.environ.get("TINYBARD_MODEL", INFERENCE_MODEL)
|
| 59 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 60 |
# ---------------------------------------------------------------------------
|
| 61 |
# Llama.cpp Inference Setup
|
| 62 |
# ---------------------------------------------------------------------------
|
|
@@ -68,17 +90,18 @@ def llm_available() -> bool:
|
|
| 68 |
"""True if we *might* succeed at an inference call (cooldown not active,
|
| 69 |
HF_TOKEN configured, model id is set)."""
|
| 70 |
import os
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
return bool(
|
| 75 |
|
| 76 |
|
| 77 |
def last_inference_status() -> dict:
|
| 78 |
"""Snapshot of the current cooldown + model for /api/model_status."""
|
| 79 |
return {
|
| 80 |
-
"model": TINYBARD_MODEL,
|
| 81 |
"cooldown": cooldown_status("tinybard"),
|
|
|
|
| 82 |
}
|
| 83 |
|
| 84 |
|
|
@@ -509,6 +532,36 @@ async def game_choice(payload: dict):
|
|
| 509 |
history=payload.get("history", []),
|
| 510 |
)
|
| 511 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 512 |
# Mount static files
|
| 513 |
fastapi_app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static")
|
| 514 |
|
|
|
|
| 19 |
import logging
|
| 20 |
import sys
|
| 21 |
from pathlib import Path
|
| 22 |
+
from typing import Dict, List, Optional
|
| 23 |
+
import threading
|
| 24 |
+
from pydantic import BaseModel
|
| 25 |
|
| 26 |
import gradio as gr
|
| 27 |
from fastapi import FastAPI
|
|
|
|
| 59 |
# Cooldown enforced in shared.inference_client.
|
| 60 |
TINYBARD_MODEL = os.environ.get("TINYBARD_MODEL", INFERENCE_MODEL)
|
| 61 |
|
| 62 |
+
# ---------------------------------------------------------------------------
|
| 63 |
+
# User-configurable inference (BYO token / model)
|
| 64 |
+
# ---------------------------------------------------------------------------
|
| 65 |
+
_USER_CONFIG_LOCK = threading.Lock()
|
| 66 |
+
_USER_CONFIG: Dict[str, Optional[str]] = {
|
| 67 |
+
"hf_token": None,
|
| 68 |
+
"model": None,
|
| 69 |
+
}
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
def get_user_hf_token() -> Optional[str]:
|
| 73 |
+
with _USER_CONFIG_LOCK:
|
| 74 |
+
return _USER_CONFIG["hf_token"]
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def get_user_model() -> Optional[str]:
|
| 78 |
+
with _USER_CONFIG_LOCK:
|
| 79 |
+
return _USER_CONFIG["model"]
|
| 80 |
+
|
| 81 |
+
|
| 82 |
# ---------------------------------------------------------------------------
|
| 83 |
# Llama.cpp Inference Setup
|
| 84 |
# ---------------------------------------------------------------------------
|
|
|
|
| 90 |
"""True if we *might* succeed at an inference call (cooldown not active,
|
| 91 |
HF_TOKEN configured, model id is set)."""
|
| 92 |
import os
|
| 93 |
+
token = get_user_hf_token() or os.environ.get("HF_TOKEN") or os.environ.get("HUGGINGFACEHUB_API_TOKEN")
|
| 94 |
+
model = get_user_model() or TINYBARD_MODEL
|
| 95 |
+
# Inference API still works anonymously for some models, so don't gate hard.
|
| 96 |
+
return bool(model) and not cooldown_active("tinybard")
|
| 97 |
|
| 98 |
|
| 99 |
def last_inference_status() -> dict:
|
| 100 |
"""Snapshot of the current cooldown + model for /api/model_status."""
|
| 101 |
return {
|
| 102 |
+
"model": get_user_model() or TINYBARD_MODEL,
|
| 103 |
"cooldown": cooldown_status("tinybard"),
|
| 104 |
+
"has_user_token": bool(get_user_hf_token()),
|
| 105 |
}
|
| 106 |
|
| 107 |
|
|
|
|
| 532 |
history=payload.get("history", []),
|
| 533 |
)
|
| 534 |
|
| 535 |
+
class UserConfig(BaseModel):
|
| 536 |
+
hf_token: Optional[str] = None
|
| 537 |
+
model: Optional[str] = None
|
| 538 |
+
|
| 539 |
+
|
| 540 |
+
@fastapi_app.post("/api/config")
|
| 541 |
+
async def update_config(cfg: UserConfig):
|
| 542 |
+
with _USER_CONFIG_LOCK:
|
| 543 |
+
if cfg.hf_token:
|
| 544 |
+
_USER_CONFIG["hf_token"] = cfg.hf_token.strip() or None
|
| 545 |
+
if cfg.model and cfg.model.strip():
|
| 546 |
+
_USER_CONFIG["model"] = cfg.model.strip()
|
| 547 |
+
current = dict(_USER_CONFIG)
|
| 548 |
+
return {
|
| 549 |
+
"status": "ok",
|
| 550 |
+
"model": current["model"] or TINYBARD_MODEL,
|
| 551 |
+
"has_token": bool(current["hf_token"]),
|
| 552 |
+
}
|
| 553 |
+
|
| 554 |
+
|
| 555 |
+
@fastapi_app.get("/api/config")
|
| 556 |
+
async def get_config():
|
| 557 |
+
with _USER_CONFIG_LOCK:
|
| 558 |
+
current = dict(_USER_CONFIG)
|
| 559 |
+
return {
|
| 560 |
+
"model": current["model"] or TINYBARD_MODEL,
|
| 561 |
+
"has_token": bool(current["hf_token"]),
|
| 562 |
+
}
|
| 563 |
+
|
| 564 |
+
|
| 565 |
# Mount static files
|
| 566 |
fastapi_app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static")
|
| 567 |
|
projects/tinybard/static/index.html
CHANGED
|
@@ -23,7 +23,7 @@
|
|
| 23 |
<div class="terminal" id="terminal">
|
| 24 |
<div class="terminal-header">
|
| 25 |
<span class="prompt">ᐴ TINYBARD v1.0 ᔔ · GIIZHI-AADIZOKED</span>
|
| 26 |
-
<span class="status" id="health-bar">NOOSISKAAZOWIN: <span id="health-val">100</span></span>
|
| 27 |
</div>
|
| 28 |
|
| 29 |
<div class="terminal-body" id="output">
|
|
@@ -69,5 +69,25 @@
|
|
| 69 |
</div>
|
| 70 |
|
| 71 |
<script type="module" src="/static/main.js"></script>
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 72 |
</body>
|
| 73 |
</html>
|
|
|
|
| 23 |
<div class="terminal" id="terminal">
|
| 24 |
<div class="terminal-header">
|
| 25 |
<span class="prompt">ᐴ TINYBARD v1.0 ᔔ · GIIZHI-AADIZOKED</span>
|
| 26 |
+
<span class="status" id="health-bar">NOOSISKAAZOWIN: <span id="health-val">100</span></span><button id="config-btn" style="float:right;background:none;border:1px solid var(--asp-amber,#FFB347);color:var(--asp-sun,#F2A93B);cursor:pointer;font-family:monospace;font-size:0.75rem;padding:0.15rem 0.4rem;border-radius:2px;">⚙ CONFIG</button>
|
| 27 |
</div>
|
| 28 |
|
| 29 |
<div class="terminal-body" id="output">
|
|
|
|
| 69 |
</div>
|
| 70 |
|
| 71 |
<script type="module" src="/static/main.js"></script>
|
| 72 |
+
|
| 73 |
+
<!-- User config modal -->
|
| 74 |
+
<div id="tb-config-modal" class="tb-modal-overlay" style="display:none;">
|
| 75 |
+
<div class="tb-modal">
|
| 76 |
+
<button class="tb-close" id="tb-config-close">✕</button>
|
| 77 |
+
<h3>⚙ Inference Configuration</h3>
|
| 78 |
+
<p style="font-size:0.8rem;color:var(--asp-moss,#588157);">
|
| 79 |
+
Provide your own HuggingFace token to unlock models that need auth.
|
| 80 |
+
Your token lives in browser memory only — never sent anywhere except api.huggingface.co.
|
| 81 |
+
</p>
|
| 82 |
+
<label for="tb-model-input">Model ID</label>
|
| 83 |
+
<input type="text" id="tb-model-input" placeholder="Qwen/Qwen2.5-1.5B-Instruct" />
|
| 84 |
+
|
| 85 |
+
<label for="tb-token-input">HF Token (optional)</label>
|
| 86 |
+
<input type="password" id="tb-token-input" placeholder="hf_..." />
|
| 87 |
+
|
| 88 |
+
<button id="tb-config-save">Save & Reload</button>
|
| 89 |
+
<span id="tb-config-status" style="margin-left:0.5rem;color:var(--asp-moss,#588157);"></span>
|
| 90 |
+
</div>
|
| 91 |
+
</div>
|
| 92 |
</body>
|
| 93 |
</html>
|
projects/tinybard/static/main.js
CHANGED
|
@@ -243,6 +243,52 @@ cmdInput.addEventListener("keydown", (e) => {
|
|
| 243 |
}
|
| 244 |
});
|
| 245 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 246 |
// Boot
|
| 247 |
// ---------------------------------------------------------------------------
|
| 248 |
(async () => {
|
|
|
|
| 243 |
}
|
| 244 |
});
|
| 245 |
|
| 246 |
+
|
| 247 |
+
// ---------------------------------------------------------------------------
|
| 248 |
+
// User Config Modal
|
| 249 |
+
// ---------------------------------------------------------------------------
|
| 250 |
+
const configBtn = document.getElementById('config-btn');
|
| 251 |
+
const configModal = document.getElementById('tb-config-modal');
|
| 252 |
+
const configClose = document.getElementById('tb-config-close');
|
| 253 |
+
const configSave = document.getElementById('tb-config-save');
|
| 254 |
+
const modelInput = document.getElementById('tb-model-input');
|
| 255 |
+
const tokenInput = document.getElementById('tb-token-input');
|
| 256 |
+
const configStatus = document.getElementById('tb-config-status');
|
| 257 |
+
|
| 258 |
+
if (configBtn && configModal) {
|
| 259 |
+
configBtn.addEventListener('click', async () => {
|
| 260 |
+
const cfg = await fetch('/api/config').then(r => r.json());
|
| 261 |
+
modelInput.value = cfg.model || '';
|
| 262 |
+
tokenInput.value = '';
|
| 263 |
+
configStatus.textContent = '';
|
| 264 |
+
configModal.style.display = 'flex';
|
| 265 |
+
});
|
| 266 |
+
|
| 267 |
+
configClose.addEventListener('click', () => {
|
| 268 |
+
configModal.style.display = 'none';
|
| 269 |
+
});
|
| 270 |
+
|
| 271 |
+
configModal.addEventListener('click', (e) => {
|
| 272 |
+
if (e.target === configModal) configModal.style.display = 'none';
|
| 273 |
+
});
|
| 274 |
+
|
| 275 |
+
configSave.addEventListener('click', async () => {
|
| 276 |
+
const body = {};
|
| 277 |
+
if (modelInput.value.trim()) body.model = modelInput.value.trim();
|
| 278 |
+
if (tokenInput.value.trim()) body.hf_token = tokenInput.value.trim();
|
| 279 |
+
|
| 280 |
+
const resp = await fetch('/api/config', {
|
| 281 |
+
method: 'POST',
|
| 282 |
+
headers: { 'Content-Type': 'application/json' },
|
| 283 |
+
body: JSON.stringify(body),
|
| 284 |
+
});
|
| 285 |
+
const data = await resp.json();
|
| 286 |
+
configStatus.textContent = data.status === 'ok' ? '✓ Saved' : '✗ Failed';
|
| 287 |
+
configStatus.style.color = data.status === 'ok' ? 'var(--asp-sun)' : 'var(--asp-ember)';
|
| 288 |
+
setTimeout(() => { configModal.style.display = 'none'; }, 800);
|
| 289 |
+
});
|
| 290 |
+
}
|
| 291 |
+
|
| 292 |
// Boot
|
| 293 |
// ---------------------------------------------------------------------------
|
| 294 |
(async () => {
|
shared/inference_client.py
CHANGED
|
@@ -143,7 +143,7 @@ def generate(
|
|
| 143 |
return InferenceResult(
|
| 144 |
text=text,
|
| 145 |
model=INFERENCE_MODEL,
|
| 146 |
-
|
| 147 |
latency_s=latency,
|
| 148 |
)
|
| 149 |
def force_clear_cooldown(project: str) -> None:
|
|
|
|
| 143 |
return InferenceResult(
|
| 144 |
text=text,
|
| 145 |
model=INFERENCE_MODEL,
|
| 146 |
+
provider=INFERENCE_PROVIDER,
|
| 147 |
latency_s=latency,
|
| 148 |
)
|
| 149 |
def force_clear_cooldown(project: str) -> None:
|