Hermes Bot commited on
Commit
ecc79cb
·
unverified ·
1 Parent(s): 9494965

feat: add user-configurable HF token + model override

Browse files

- Add /api/config GET/POST endpoints for BYO HF token + model id
- Token stored in-memory only, never written to disk
- UI: config button + modal in CRT terminal footer
- Frontend: localStorage-free, all state server-side per-session
- Procedural fallback always active when inference fails or cooldown hits

projects/tinybard/app.py CHANGED
@@ -19,7 +19,9 @@ import random
19
  import logging
20
  import sys
21
  from pathlib import Path
22
- from typing import Dict, List
 
 
23
 
24
  import gradio as gr
25
  from fastapi import FastAPI
@@ -57,6 +59,26 @@ STATIC_DIR = BASE_DIR / "static"
57
  # Cooldown enforced in shared.inference_client.
58
  TINYBARD_MODEL = os.environ.get("TINYBARD_MODEL", INFERENCE_MODEL)
59
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
60
  # ---------------------------------------------------------------------------
61
  # Llama.cpp Inference Setup
62
  # ---------------------------------------------------------------------------
@@ -68,17 +90,18 @@ def llm_available() -> bool:
68
  """True if we *might* succeed at an inference call (cooldown not active,
69
  HF_TOKEN configured, model id is set)."""
70
  import os
71
- if not os.environ.get("HF_TOKEN") and not os.environ.get("HUGGINGFACEHUB_API_TOKEN"):
72
- # Inference API still works anonymously for some models, so don't gate hard.
73
- pass
74
- return bool(TINYBARD_MODEL) and not cooldown_active("tinybard")
75
 
76
 
77
  def last_inference_status() -> dict:
78
  """Snapshot of the current cooldown + model for /api/model_status."""
79
  return {
80
- "model": TINYBARD_MODEL,
81
  "cooldown": cooldown_status("tinybard"),
 
82
  }
83
 
84
 
@@ -509,6 +532,36 @@ async def game_choice(payload: dict):
509
  history=payload.get("history", []),
510
  )
511
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
512
  # Mount static files
513
  fastapi_app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static")
514
 
 
19
  import logging
20
  import sys
21
  from pathlib import Path
22
+ from typing import Dict, List, Optional
23
+ import threading
24
+ from pydantic import BaseModel
25
 
26
  import gradio as gr
27
  from fastapi import FastAPI
 
59
  # Cooldown enforced in shared.inference_client.
60
  TINYBARD_MODEL = os.environ.get("TINYBARD_MODEL", INFERENCE_MODEL)
61
 
62
+ # ---------------------------------------------------------------------------
63
+ # User-configurable inference (BYO token / model)
64
+ # ---------------------------------------------------------------------------
65
+ _USER_CONFIG_LOCK = threading.Lock()
66
+ _USER_CONFIG: Dict[str, Optional[str]] = {
67
+ "hf_token": None,
68
+ "model": None,
69
+ }
70
+
71
+
72
+ def get_user_hf_token() -> Optional[str]:
73
+ with _USER_CONFIG_LOCK:
74
+ return _USER_CONFIG["hf_token"]
75
+
76
+
77
+ def get_user_model() -> Optional[str]:
78
+ with _USER_CONFIG_LOCK:
79
+ return _USER_CONFIG["model"]
80
+
81
+
82
  # ---------------------------------------------------------------------------
83
  # Llama.cpp Inference Setup
84
  # ---------------------------------------------------------------------------
 
90
  """True if we *might* succeed at an inference call (cooldown not active,
91
  HF_TOKEN configured, model id is set)."""
92
  import os
93
+ token = get_user_hf_token() or os.environ.get("HF_TOKEN") or os.environ.get("HUGGINGFACEHUB_API_TOKEN")
94
+ model = get_user_model() or TINYBARD_MODEL
95
+ # Inference API still works anonymously for some models, so don't gate hard.
96
+ return bool(model) and not cooldown_active("tinybard")
97
 
98
 
99
  def last_inference_status() -> dict:
100
  """Snapshot of the current cooldown + model for /api/model_status."""
101
  return {
102
+ "model": get_user_model() or TINYBARD_MODEL,
103
  "cooldown": cooldown_status("tinybard"),
104
+ "has_user_token": bool(get_user_hf_token()),
105
  }
106
 
107
 
 
532
  history=payload.get("history", []),
533
  )
534
 
535
+ class UserConfig(BaseModel):
536
+ hf_token: Optional[str] = None
537
+ model: Optional[str] = None
538
+
539
+
540
+ @fastapi_app.post("/api/config")
541
+ async def update_config(cfg: UserConfig):
542
+ with _USER_CONFIG_LOCK:
543
+ if cfg.hf_token:
544
+ _USER_CONFIG["hf_token"] = cfg.hf_token.strip() or None
545
+ if cfg.model and cfg.model.strip():
546
+ _USER_CONFIG["model"] = cfg.model.strip()
547
+ current = dict(_USER_CONFIG)
548
+ return {
549
+ "status": "ok",
550
+ "model": current["model"] or TINYBARD_MODEL,
551
+ "has_token": bool(current["hf_token"]),
552
+ }
553
+
554
+
555
+ @fastapi_app.get("/api/config")
556
+ async def get_config():
557
+ with _USER_CONFIG_LOCK:
558
+ current = dict(_USER_CONFIG)
559
+ return {
560
+ "model": current["model"] or TINYBARD_MODEL,
561
+ "has_token": bool(current["hf_token"]),
562
+ }
563
+
564
+
565
  # Mount static files
566
  fastapi_app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static")
567
 
projects/tinybard/static/index.html CHANGED
@@ -23,7 +23,7 @@
23
  <div class="terminal" id="terminal">
24
  <div class="terminal-header">
25
  <span class="prompt">ᐴ TINYBARD v1.0 ᔔ · GIIZHI-AADIZOKED</span>
26
- <span class="status" id="health-bar">NOOSISKAAZOWIN: <span id="health-val">100</span></span>
27
  </div>
28
 
29
  <div class="terminal-body" id="output">
@@ -69,5 +69,25 @@
69
  </div>
70
 
71
  <script type="module" src="/static/main.js"></script>
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
72
  </body>
73
  </html>
 
23
  <div class="terminal" id="terminal">
24
  <div class="terminal-header">
25
  <span class="prompt">ᐴ TINYBARD v1.0 ᔔ · GIIZHI-AADIZOKED</span>
26
+ <span class="status" id="health-bar">NOOSISKAAZOWIN: <span id="health-val">100</span></span><button id="config-btn" style="float:right;background:none;border:1px solid var(--asp-amber,#FFB347);color:var(--asp-sun,#F2A93B);cursor:pointer;font-family:monospace;font-size:0.75rem;padding:0.15rem 0.4rem;border-radius:2px;">⚙ CONFIG</button>
27
  </div>
28
 
29
  <div class="terminal-body" id="output">
 
69
  </div>
70
 
71
  <script type="module" src="/static/main.js"></script>
72
+
73
+ <!-- User config modal -->
74
+ <div id="tb-config-modal" class="tb-modal-overlay" style="display:none;">
75
+ <div class="tb-modal">
76
+ <button class="tb-close" id="tb-config-close">✕</button>
77
+ <h3>⚙ Inference Configuration</h3>
78
+ <p style="font-size:0.8rem;color:var(--asp-moss,#588157);">
79
+ Provide your own HuggingFace token to unlock models that need auth.
80
+ Your token lives in browser memory only — never sent anywhere except api.huggingface.co.
81
+ </p>
82
+ <label for="tb-model-input">Model ID</label>
83
+ <input type="text" id="tb-model-input" placeholder="Qwen/Qwen2.5-1.5B-Instruct" />
84
+
85
+ <label for="tb-token-input">HF Token (optional)</label>
86
+ <input type="password" id="tb-token-input" placeholder="hf_..." />
87
+
88
+ <button id="tb-config-save">Save & Reload</button>
89
+ <span id="tb-config-status" style="margin-left:0.5rem;color:var(--asp-moss,#588157);"></span>
90
+ </div>
91
+ </div>
92
  </body>
93
  </html>
projects/tinybard/static/main.js CHANGED
@@ -243,6 +243,52 @@ cmdInput.addEventListener("keydown", (e) => {
243
  }
244
  });
245
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
246
  // Boot
247
  // ---------------------------------------------------------------------------
248
  (async () => {
 
243
  }
244
  });
245
 
246
+
247
+ // ---------------------------------------------------------------------------
248
+ // User Config Modal
249
+ // ---------------------------------------------------------------------------
250
+ const configBtn = document.getElementById('config-btn');
251
+ const configModal = document.getElementById('tb-config-modal');
252
+ const configClose = document.getElementById('tb-config-close');
253
+ const configSave = document.getElementById('tb-config-save');
254
+ const modelInput = document.getElementById('tb-model-input');
255
+ const tokenInput = document.getElementById('tb-token-input');
256
+ const configStatus = document.getElementById('tb-config-status');
257
+
258
+ if (configBtn && configModal) {
259
+ configBtn.addEventListener('click', async () => {
260
+ const cfg = await fetch('/api/config').then(r => r.json());
261
+ modelInput.value = cfg.model || '';
262
+ tokenInput.value = '';
263
+ configStatus.textContent = '';
264
+ configModal.style.display = 'flex';
265
+ });
266
+
267
+ configClose.addEventListener('click', () => {
268
+ configModal.style.display = 'none';
269
+ });
270
+
271
+ configModal.addEventListener('click', (e) => {
272
+ if (e.target === configModal) configModal.style.display = 'none';
273
+ });
274
+
275
+ configSave.addEventListener('click', async () => {
276
+ const body = {};
277
+ if (modelInput.value.trim()) body.model = modelInput.value.trim();
278
+ if (tokenInput.value.trim()) body.hf_token = tokenInput.value.trim();
279
+
280
+ const resp = await fetch('/api/config', {
281
+ method: 'POST',
282
+ headers: { 'Content-Type': 'application/json' },
283
+ body: JSON.stringify(body),
284
+ });
285
+ const data = await resp.json();
286
+ configStatus.textContent = data.status === 'ok' ? '✓ Saved' : '✗ Failed';
287
+ configStatus.style.color = data.status === 'ok' ? 'var(--asp-sun)' : 'var(--asp-ember)';
288
+ setTimeout(() => { configModal.style.display = 'none'; }, 800);
289
+ });
290
+ }
291
+
292
  // Boot
293
  // ---------------------------------------------------------------------------
294
  (async () => {
shared/inference_client.py CHANGED
@@ -143,7 +143,7 @@ def generate(
143
  return InferenceResult(
144
  text=text,
145
  model=INFERENCE_MODEL,
146
-
147
  latency_s=latency,
148
  )
149
  def force_clear_cooldown(project: str) -> None:
 
143
  return InferenceResult(
144
  text=text,
145
  model=INFERENCE_MODEL,
146
+ provider=INFERENCE_PROVIDER,
147
  latency_s=latency,
148
  )
149
  def force_clear_cooldown(project: str) -> None: