Spaces:
Running
feat(api): /api/js-apps with LLM-inferred categories
Browse filesAdd a curated route that filters the catalog to JS-embeddable apps
(`reachy_mini_js_app` tag) and enriches each entry with categories
inferred by Llama-3.1-8B-Instruct via HF Inference Providers, from
a closed taxonomy of 8 slugs (music, dance, voice, storytelling,
kids, vision, companion, dev-tools).
Inference results are persisted in a HF dataset
(`tfrere/reachy-mini-app-categories` by default, override via
HF_CATEGORIES_DATASET) and re-loaded at boot, so cold starts only
re-classify what actually changed (lastModified-keyed staleness).
Pre-existing /api/apps payload is unchanged: this is purely
additive. Mobile shell can adopt /api/js-apps incrementally.
- server/categories.js: 8-slug taxonomy + LLM-friendly descriptions
- server/categorize.js: README fetch/clean + chat-completion call
- server/categoryCache.js: in-mem map + dataset commit/load
- server/index.js: route, warmup batch, manual refresh endpoint
- .env.example: HF_TOKEN + HF_CATEGORIES_DATASET docs
- .gitignore: ignore local .env
Co-authored-by: Cursor <cursoragent@cursor.com>
- .env.example +35 -0
- .gitignore +2 -0
- server/categories.js +145 -0
- server/categorize.js +292 -0
- server/categoryCache.js +290 -0
- server/index.js +282 -1
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Reachy Mini Website server env vars
|
| 2 |
+
#
|
| 3 |
+
# Copy this file to `.env` and fill in the values for local dev.
|
| 4 |
+
# In production (HF Space), set these from the Space's "Settings →
|
| 5 |
+
# Variables and secrets" panel, NOT from a committed `.env`.
|
| 6 |
+
# (`.env` is gitignored.)
|
| 7 |
+
|
| 8 |
+
# -----------------------------------------------------------------------------
|
| 9 |
+
# Server
|
| 10 |
+
# -----------------------------------------------------------------------------
|
| 11 |
+
# Port the Express server listens on. Defaults to 7860 (HF Space convention).
|
| 12 |
+
# PORT=7860
|
| 13 |
+
|
| 14 |
+
# -----------------------------------------------------------------------------
|
| 15 |
+
# OAuth (used by /api/oauth-config and the in-iframe sign-in flow)
|
| 16 |
+
# -----------------------------------------------------------------------------
|
| 17 |
+
# Set in the Space when `hf_oauth: true` is in README.md.
|
| 18 |
+
# OAUTH_CLIENT_ID=
|
| 19 |
+
# OAUTH_SCOPES=openid profile
|
| 20 |
+
|
| 21 |
+
# -----------------------------------------------------------------------------
|
| 22 |
+
# HF Inference Providers (used by /api/js-apps category inference)
|
| 23 |
+
# -----------------------------------------------------------------------------
|
| 24 |
+
# Required for category inference. A standard READ token is enough -
|
| 25 |
+
# Inference Providers access is on by default for FREE/PRO tokens.
|
| 26 |
+
# Without this, /api/js-apps still works but every entry will have
|
| 27 |
+
# `categories: null` (the route logs a warning at startup).
|
| 28 |
+
HF_TOKEN=
|
| 29 |
+
|
| 30 |
+
# Dataset where the inferred-categories cache is persisted.
|
| 31 |
+
# Defaults to `tfrere/reachy-mini-app-categories` (per-user namespace,
|
| 32 |
+
# auto-created on first commit). Override to e.g.
|
| 33 |
+
# `pollen-robotics/reachy-mini-app-categories` once the org dataset
|
| 34 |
+
# exists and the HF_TOKEN has write access to it.
|
| 35 |
+
# HF_CATEGORIES_DATASET=tfrere/reachy-mini-app-categories
|
|
@@ -22,3 +22,5 @@ dist-ssr
|
|
| 22 |
*.njsproj
|
| 23 |
*.sln
|
| 24 |
*.sw?
|
|
|
|
|
|
|
|
|
| 22 |
*.njsproj
|
| 23 |
*.sln
|
| 24 |
*.sw?
|
| 25 |
+
|
| 26 |
+
.env
|
|
@@ -0,0 +1,145 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/**
|
| 2 |
+
* Predefined taxonomy for JS Reachy Mini apps.
|
| 3 |
+
*
|
| 4 |
+
* These slugs are the ONLY valid output values for the LLM
|
| 5 |
+
* inference step (anything else is dropped at parse time) and
|
| 6 |
+
* the values consumers (mobile shell, website) filter on.
|
| 7 |
+
*
|
| 8 |
+
* Why a closed list instead of free-form tags
|
| 9 |
+
* ──────────────────────────────────────────
|
| 10 |
+
* The HF Spaces catalog has no usable categorization for the
|
| 11 |
+
* reachy_mini_js_app subset (only platform/SDK tags). We bridge
|
| 12 |
+
* the gap by inferring categories with an LLM, but we have to
|
| 13 |
+
* constrain the model's output: a closed list keeps category
|
| 14 |
+
* pages stable, lets us pre-pick emojis/labels, and avoids the
|
| 15 |
+
* "30 near-duplicate slugs" problem you'd get with free-form.
|
| 16 |
+
*
|
| 17 |
+
* Bumping the taxonomy
|
| 18 |
+
* ────────────────────
|
| 19 |
+
* Adding, removing or renaming a slug changes the meaning of
|
| 20 |
+
* cached entries. Bump TAXONOMY_VERSION when you do that: the
|
| 21 |
+
* cache layer compares each entry's `taxonomyVersion` against
|
| 22 |
+
* the live one and recomputes stale ones on the next pass.
|
| 23 |
+
*/
|
| 24 |
+
|
| 25 |
+
export const TAXONOMY_VERSION = 1;
|
| 26 |
+
|
| 27 |
+
/**
|
| 28 |
+
* Canonical category list. Keep slugs short, kebab-case, and
|
| 29 |
+
* memorable: they end up in URLs (e.g. `?cat=music`) and in
|
| 30 |
+
* filter chips on mobile.
|
| 31 |
+
*
|
| 32 |
+
* The `description` field is the SOLE source of truth the LLM
|
| 33 |
+
* sees - keep them factual, scope-bounded, and example-led so
|
| 34 |
+
* the model has signal for both inclusion and exclusion.
|
| 35 |
+
*/
|
| 36 |
+
export const CATEGORIES = [
|
| 37 |
+
{
|
| 38 |
+
slug: 'music',
|
| 39 |
+
label: 'Music & Beats',
|
| 40 |
+
emoji: '🎵',
|
| 41 |
+
description:
|
| 42 |
+
'Apps where Reachy plays, mixes, generates, or reacts to music ' +
|
| 43 |
+
'(DJ sets, beat-makers, blind-tests). NOT for apps that just ' +
|
| 44 |
+
'happen to use audio cues.',
|
| 45 |
+
},
|
| 46 |
+
{
|
| 47 |
+
slug: 'dance',
|
| 48 |
+
label: 'Dance & Motion',
|
| 49 |
+
emoji: '💃',
|
| 50 |
+
description:
|
| 51 |
+
'Apps centered on dance choreographies, motion replay, ' +
|
| 52 |
+
'kinetic shows, or recording/replaying robot movements.',
|
| 53 |
+
},
|
| 54 |
+
{
|
| 55 |
+
slug: 'voice',
|
| 56 |
+
label: 'Voice & Conversation',
|
| 57 |
+
emoji: '🗣️',
|
| 58 |
+
description:
|
| 59 |
+
'Apps where Reachy talks, listens, or holds a real-time voice ' +
|
| 60 |
+
'conversation: TTS players, LLM-driven chat (OpenAI Realtime, ' +
|
| 61 |
+
'Claude, Perplexity, etc.), wake-word demos.',
|
| 62 |
+
},
|
| 63 |
+
{
|
| 64 |
+
slug: 'storytelling',
|
| 65 |
+
label: 'Stories',
|
| 66 |
+
emoji: '📖',
|
| 67 |
+
description:
|
| 68 |
+
'Apps that tell, generate, or guide narrative stories ' +
|
| 69 |
+
'(interactive fiction, bedtime stories, narrated adventures).',
|
| 70 |
+
},
|
| 71 |
+
{
|
| 72 |
+
slug: 'kids',
|
| 73 |
+
label: 'For Kids',
|
| 74 |
+
emoji: '🧒',
|
| 75 |
+
description:
|
| 76 |
+
'Apps explicitly designed for children: encyclopedias, ' +
|
| 77 |
+
'kid-friendly stories, learning games, simple Q&A.',
|
| 78 |
+
},
|
| 79 |
+
{
|
| 80 |
+
slug: 'vision',
|
| 81 |
+
label: 'Vision & Camera',
|
| 82 |
+
emoji: '👁️',
|
| 83 |
+
description:
|
| 84 |
+
"Apps that consume Reachy's camera feed: face/hand tracking, " +
|
| 85 |
+
'image classification, mimicry, gesture detection.',
|
| 86 |
+
},
|
| 87 |
+
{
|
| 88 |
+
slug: 'companion',
|
| 89 |
+
label: 'Companion',
|
| 90 |
+
emoji: '🤝',
|
| 91 |
+
description:
|
| 92 |
+
'Apps positioned as life/emotional companions, mood support, ' +
|
| 93 |
+
'or friendly buddies that develop a personality over time.',
|
| 94 |
+
},
|
| 95 |
+
{
|
| 96 |
+
slug: 'dev-tools',
|
| 97 |
+
label: 'Dev & Demos',
|
| 98 |
+
emoji: '🛠️',
|
| 99 |
+
description:
|
| 100 |
+
'Technical demos, debugging tools, remote-control utilities, ' +
|
| 101 |
+
'minimal proof-of-concepts, and dev-only experiments meant to ' +
|
| 102 |
+
'showcase a single API or behaviour.',
|
| 103 |
+
},
|
| 104 |
+
];
|
| 105 |
+
|
| 106 |
+
export const ALLOWED_SLUGS = new Set(CATEGORIES.map((c) => c.slug));
|
| 107 |
+
|
| 108 |
+
export function isValidSlug(slug) {
|
| 109 |
+
return ALLOWED_SLUGS.has(slug);
|
| 110 |
+
}
|
| 111 |
+
|
| 112 |
+
/**
|
| 113 |
+
* Render the taxonomy as a bulleted list for the LLM prompt.
|
| 114 |
+
* Format mirrors what the model is asked to output (slug first)
|
| 115 |
+
* to nudge it towards copying the exact string back.
|
| 116 |
+
*/
|
| 117 |
+
export function buildLlmCategoryList() {
|
| 118 |
+
return CATEGORIES.map((c) => `- ${c.slug}: ${c.description}`).join('\n');
|
| 119 |
+
}
|
| 120 |
+
|
| 121 |
+
/**
|
| 122 |
+
* Sanitize a raw LLM-returned list of slugs:
|
| 123 |
+
* - drop non-strings
|
| 124 |
+
* - lowercase + trim
|
| 125 |
+
* - drop unknown slugs (hallucinations)
|
| 126 |
+
* - dedupe while preserving order (the model orders by relevance)
|
| 127 |
+
* - cap to MAX_CATEGORIES
|
| 128 |
+
*
|
| 129 |
+
* Returns a fresh array; never mutates input.
|
| 130 |
+
*/
|
| 131 |
+
export function sanitizeSlugs(raw, maxCategories = 3) {
|
| 132 |
+
if (!Array.isArray(raw)) return [];
|
| 133 |
+
const seen = new Set();
|
| 134 |
+
const out = [];
|
| 135 |
+
for (const v of raw) {
|
| 136 |
+
if (typeof v !== 'string') continue;
|
| 137 |
+
const slug = v.trim().toLowerCase();
|
| 138 |
+
if (!slug || seen.has(slug)) continue;
|
| 139 |
+
if (!ALLOWED_SLUGS.has(slug)) continue;
|
| 140 |
+
seen.add(slug);
|
| 141 |
+
out.push(slug);
|
| 142 |
+
if (out.length >= maxCategories) break;
|
| 143 |
+
}
|
| 144 |
+
return out;
|
| 145 |
+
}
|
|
@@ -0,0 +1,292 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/**
|
| 2 |
+
* LLM-based category inference for JS Reachy Mini apps.
|
| 3 |
+
*
|
| 4 |
+
* Pipeline (`categorizeApp`)
|
| 5 |
+
* ──────────────────────────
|
| 6 |
+
* 1. Fetch the Space's README from HF Hub (raw)
|
| 7 |
+
* 2. Strip frontmatter, images, badges, raw HTML, then truncate
|
| 8 |
+
* 3. Call a chat LLM via HF Inference Providers (OpenAI-compatible)
|
| 9 |
+
* with the predefined taxonomy + the app's name/description
|
| 10 |
+
* 4. Parse JSON, validate against ALLOWED_SLUGS, keep up to 3
|
| 11 |
+
*
|
| 12 |
+
* Robustness contract
|
| 13 |
+
* ───────────────────
|
| 14 |
+
* `categorizeApp` NEVER throws on transient failure (network,
|
| 15 |
+
* 429, malformed JSON). It returns `null`, which the cache layer
|
| 16 |
+
* interprets as "not yet categorized; retry on the next pass".
|
| 17 |
+
* Hard errors (HF_TOKEN missing) are signalled by a thrown
|
| 18 |
+
* `HfTokenMissingError` so the caller can short-circuit the
|
| 19 |
+
* whole batch.
|
| 20 |
+
*/
|
| 21 |
+
|
| 22 |
+
import {
|
| 23 |
+
buildLlmCategoryList,
|
| 24 |
+
sanitizeSlugs,
|
| 25 |
+
} from './categories.js';
|
| 26 |
+
|
| 27 |
+
// HF Inference Providers - OpenAI-compatible router. Auto-routes
|
| 28 |
+
// the request to whichever provider currently serves the model
|
| 29 |
+
// (Together, Nebius, Fireworks, Sambanova...). The token must
|
| 30 |
+
// have `Inference Providers` access (default for all PRO and
|
| 31 |
+
// most FREE tokens since 2025).
|
| 32 |
+
const HF_INFERENCE_URL = 'https://router.huggingface.co/v1/chat/completions';
|
| 33 |
+
|
| 34 |
+
// 8B model: cheap, fast (~1 s per call), more than enough for a
|
| 35 |
+
// closed-list multi-label classification with good descriptions.
|
| 36 |
+
// If quality drifts we can swap to 70B without touching anything
|
| 37 |
+
// else - the prompt is generic.
|
| 38 |
+
const DEFAULT_MODEL = 'meta-llama/Llama-3.1-8B-Instruct';
|
| 39 |
+
|
| 40 |
+
// README budget
|
| 41 |
+
const README_MAX_CHARS = 3000;
|
| 42 |
+
const MAX_CATEGORIES_PER_APP = 3;
|
| 43 |
+
|
| 44 |
+
// LLM call budget
|
| 45 |
+
const LLM_TIMEOUT_MS = 30_000;
|
| 46 |
+
const LLM_MAX_TOKENS = 120;
|
| 47 |
+
const LLM_TEMPERATURE = 0;
|
| 48 |
+
|
| 49 |
+
export class HfTokenMissingError extends Error {
|
| 50 |
+
constructor() {
|
| 51 |
+
super('HF_TOKEN env var is not set; cannot call HF Inference Providers.');
|
| 52 |
+
this.name = 'HfTokenMissingError';
|
| 53 |
+
}
|
| 54 |
+
}
|
| 55 |
+
|
| 56 |
+
/**
|
| 57 |
+
* Fetch a Space's README from HF Hub. Returns the raw markdown
|
| 58 |
+
* string, or `null` if the request fails (404, network, etc.) -
|
| 59 |
+
* the caller falls back to "name + description only" in that case,
|
| 60 |
+
* which is still enough signal for the LLM on most apps.
|
| 61 |
+
*/
|
| 62 |
+
export async function fetchSpaceReadme(spaceId, { signal } = {}) {
|
| 63 |
+
if (!spaceId || typeof spaceId !== 'string') return null;
|
| 64 |
+
// The README of a HF Space lives at /spaces/<id>/raw/main/README.md.
|
| 65 |
+
// The `raw` endpoint returns the file as-is (no Hub UI wrapping)
|
| 66 |
+
// and is anonymous-friendly, so no auth is needed here.
|
| 67 |
+
const url = `https://huggingface.co/spaces/${spaceId}/raw/main/README.md`;
|
| 68 |
+
try {
|
| 69 |
+
const res = await fetch(url, { signal });
|
| 70 |
+
if (!res.ok) return null;
|
| 71 |
+
return await res.text();
|
| 72 |
+
} catch {
|
| 73 |
+
return null;
|
| 74 |
+
}
|
| 75 |
+
}
|
| 76 |
+
|
| 77 |
+
/**
|
| 78 |
+
* Lightly clean a raw README so the LLM doesn't burn tokens on
|
| 79 |
+
* boilerplate (HF frontmatter, badges, images) and so the actual
|
| 80 |
+
* prose surfaces above the truncation budget.
|
| 81 |
+
*
|
| 82 |
+
* We keep transformations conservative: we never edit the
|
| 83 |
+
* surrounding prose, we just delete decorative tokens. Anything
|
| 84 |
+
* cosmetic-only that clearly isn't signal for classification
|
| 85 |
+
* (badges, images, raw HTML).
|
| 86 |
+
*/
|
| 87 |
+
export function cleanReadme(raw) {
|
| 88 |
+
if (!raw || typeof raw !== 'string') return '';
|
| 89 |
+
let txt = raw;
|
| 90 |
+
|
| 91 |
+
// 1. Strip the YAML frontmatter at the very top (HF Spaces
|
| 92 |
+
// ship a mandatory `---\n...metadata...\n---` block whose
|
| 93 |
+
// fields are already exposed to us via the catalog payload,
|
| 94 |
+
// so feeding them to the LLM is pure noise).
|
| 95 |
+
txt = txt.replace(/^---\n[\s\S]*?\n---\n?/, '');
|
| 96 |
+
|
| 97 |
+
// 2. Drop image markdown (``) and HTML <img> tags.
|
| 98 |
+
// Vision apps tend to load up READMEs with screenshots and
|
| 99 |
+
// GIFs; the alt text is sometimes useful but more often it's
|
| 100 |
+
// "demo.gif" - low signal/noise ratio.
|
| 101 |
+
txt = txt.replace(/!\[[^\]]*\]\([^)]+\)/g, '');
|
| 102 |
+
txt = txt.replace(/<img\b[^>]*>/gi, '');
|
| 103 |
+
|
| 104 |
+
// 3. Strip shields.io / GitHub badges (markdown links that
|
| 105 |
+
// wrap an image). They survive (2) only when nested.
|
| 106 |
+
txt = txt.replace(/\[!\[[^\]]*\]\([^)]+\)\]\([^)]+\)/g, '');
|
| 107 |
+
|
| 108 |
+
// 4. Generic HTML stripping. Most READMEs are pure markdown,
|
| 109 |
+
// but some authors embed `<details>`, `<sub>`, `<center>`
|
| 110 |
+
// blocks. Keep the inner text, drop the tags.
|
| 111 |
+
txt = txt.replace(/<\/?[a-zA-Z][^>]*>/g, '');
|
| 112 |
+
|
| 113 |
+
// 5. Collapse runs of blank lines so trimming doesn't waste
|
| 114 |
+
// tokens on the gap.
|
| 115 |
+
txt = txt.replace(/\n{3,}/g, '\n\n');
|
| 116 |
+
|
| 117 |
+
// 6. Truncate. We slice at the paragraph boundary closest to
|
| 118 |
+
// the budget so we don't end mid-sentence.
|
| 119 |
+
if (txt.length > README_MAX_CHARS) {
|
| 120 |
+
const cut = txt.lastIndexOf('\n\n', README_MAX_CHARS);
|
| 121 |
+
txt = txt.slice(0, cut > README_MAX_CHARS / 2 ? cut : README_MAX_CHARS);
|
| 122 |
+
}
|
| 123 |
+
|
| 124 |
+
return txt.trim();
|
| 125 |
+
}
|
| 126 |
+
|
| 127 |
+
/**
|
| 128 |
+
* Build the chat messages handed to the LLM. The system prompt
|
| 129 |
+
* pins the closed taxonomy + the output shape; the user prompt
|
| 130 |
+
* is the app context.
|
| 131 |
+
*
|
| 132 |
+
* We deliberately spell out the JSON-only constraint twice
|
| 133 |
+
* (system + user) because mid-tier 8B models tend to drift into
|
| 134 |
+
* conversational replies on classification tasks otherwise.
|
| 135 |
+
*/
|
| 136 |
+
function buildMessages({ name, description, readme }) {
|
| 137 |
+
const taxonomy = buildLlmCategoryList();
|
| 138 |
+
const system = [
|
| 139 |
+
'You classify a Reachy Mini robot app into a closed list of categories.',
|
| 140 |
+
'',
|
| 141 |
+
'Output ONLY a single JSON object with this exact shape:',
|
| 142 |
+
'{"categories": ["slug1", "slug2"]}',
|
| 143 |
+
'',
|
| 144 |
+
`Pick 1 to ${MAX_CATEGORIES_PER_APP} categories from the list below, ` +
|
| 145 |
+
'ordered from most to least relevant. Use the EXACT slug.',
|
| 146 |
+
'If none of the categories clearly fits, output {"categories": []}.',
|
| 147 |
+
'',
|
| 148 |
+
'Available categories:',
|
| 149 |
+
taxonomy,
|
| 150 |
+
'',
|
| 151 |
+
'Do not include any text outside the JSON object. No prose, no code fences.',
|
| 152 |
+
].join('\n');
|
| 153 |
+
|
| 154 |
+
const user = [
|
| 155 |
+
`App name: ${name || '(unknown)'}`,
|
| 156 |
+
`Short description: ${description || '(none)'}`,
|
| 157 |
+
'',
|
| 158 |
+
'README excerpt:',
|
| 159 |
+
readme || '(no README available)',
|
| 160 |
+
'',
|
| 161 |
+
'Return the JSON now.',
|
| 162 |
+
].join('\n');
|
| 163 |
+
|
| 164 |
+
return [
|
| 165 |
+
{ role: 'system', content: system },
|
| 166 |
+
{ role: 'user', content: user },
|
| 167 |
+
];
|
| 168 |
+
}
|
| 169 |
+
|
| 170 |
+
/**
|
| 171 |
+
* Best-effort JSON extraction. Some 8B models still wrap the
|
| 172 |
+
* answer in ``` fences or prepend "Sure, here you go:". We grab
|
| 173 |
+
* the first balanced `{...}` block and parse that.
|
| 174 |
+
*/
|
| 175 |
+
function extractJsonObject(text) {
|
| 176 |
+
if (!text || typeof text !== 'string') return null;
|
| 177 |
+
const start = text.indexOf('{');
|
| 178 |
+
if (start === -1) return null;
|
| 179 |
+
let depth = 0;
|
| 180 |
+
for (let i = start; i < text.length; i++) {
|
| 181 |
+
const ch = text[i];
|
| 182 |
+
if (ch === '{') depth++;
|
| 183 |
+
else if (ch === '}') {
|
| 184 |
+
depth--;
|
| 185 |
+
if (depth === 0) {
|
| 186 |
+
const slice = text.slice(start, i + 1);
|
| 187 |
+
try {
|
| 188 |
+
return JSON.parse(slice);
|
| 189 |
+
} catch {
|
| 190 |
+
return null;
|
| 191 |
+
}
|
| 192 |
+
}
|
| 193 |
+
}
|
| 194 |
+
}
|
| 195 |
+
return null;
|
| 196 |
+
}
|
| 197 |
+
|
| 198 |
+
/**
|
| 199 |
+
* Call the HF Inference Providers chat endpoint. Returns the
|
| 200 |
+
* raw assistant message string, or `null` on any error.
|
| 201 |
+
*/
|
| 202 |
+
async function callLlm({ messages, model, signal }) {
|
| 203 |
+
const token = process.env.HF_TOKEN;
|
| 204 |
+
if (!token) throw new HfTokenMissingError();
|
| 205 |
+
|
| 206 |
+
const body = {
|
| 207 |
+
model,
|
| 208 |
+
messages,
|
| 209 |
+
temperature: LLM_TEMPERATURE,
|
| 210 |
+
max_tokens: LLM_MAX_TOKENS,
|
| 211 |
+
// `response_format` is honoured by some providers (Nebius,
|
| 212 |
+
// Together) but ignored by others. It's a free upgrade when
|
| 213 |
+
// present, harmless otherwise; the JSON-extractor below is
|
| 214 |
+
// the real safety net.
|
| 215 |
+
response_format: { type: 'json_object' },
|
| 216 |
+
};
|
| 217 |
+
|
| 218 |
+
let res;
|
| 219 |
+
try {
|
| 220 |
+
res = await fetch(HF_INFERENCE_URL, {
|
| 221 |
+
method: 'POST',
|
| 222 |
+
headers: {
|
| 223 |
+
'Authorization': `Bearer ${token}`,
|
| 224 |
+
'Content-Type': 'application/json',
|
| 225 |
+
},
|
| 226 |
+
body: JSON.stringify(body),
|
| 227 |
+
signal,
|
| 228 |
+
});
|
| 229 |
+
} catch (err) {
|
| 230 |
+
console.warn(`[categorize] LLM fetch failed: ${err.message}`);
|
| 231 |
+
return null;
|
| 232 |
+
}
|
| 233 |
+
|
| 234 |
+
if (!res.ok) {
|
| 235 |
+
const detail = await res.text().catch(() => '');
|
| 236 |
+
console.warn(
|
| 237 |
+
`[categorize] LLM HTTP ${res.status}: ${detail.slice(0, 200)}`,
|
| 238 |
+
);
|
| 239 |
+
return null;
|
| 240 |
+
}
|
| 241 |
+
|
| 242 |
+
let json;
|
| 243 |
+
try {
|
| 244 |
+
json = await res.json();
|
| 245 |
+
} catch {
|
| 246 |
+
return null;
|
| 247 |
+
}
|
| 248 |
+
return json?.choices?.[0]?.message?.content ?? null;
|
| 249 |
+
}
|
| 250 |
+
|
| 251 |
+
/**
|
| 252 |
+
* Public entry point.
|
| 253 |
+
*
|
| 254 |
+
* Returns a string[] of validated slugs (0-3 items), or `null`
|
| 255 |
+
* on transient failure so the caller can mark the entry "needs
|
| 256 |
+
* retry" without writing a misleading empty list.
|
| 257 |
+
*
|
| 258 |
+
* Treat an empty array `[]` as "the LLM looked and concluded
|
| 259 |
+
* none fit" - that's a valid, cacheable outcome.
|
| 260 |
+
*/
|
| 261 |
+
export async function categorizeApp({
|
| 262 |
+
name,
|
| 263 |
+
description,
|
| 264 |
+
spaceId,
|
| 265 |
+
model = DEFAULT_MODEL,
|
| 266 |
+
} = {}) {
|
| 267 |
+
if (!spaceId) return null;
|
| 268 |
+
|
| 269 |
+
const ctrl = new AbortController();
|
| 270 |
+
const timeoutId = setTimeout(() => ctrl.abort(), LLM_TIMEOUT_MS);
|
| 271 |
+
|
| 272 |
+
try {
|
| 273 |
+
const rawReadme = await fetchSpaceReadme(spaceId, { signal: ctrl.signal });
|
| 274 |
+
const readme = cleanReadme(rawReadme);
|
| 275 |
+
|
| 276 |
+
const messages = buildMessages({ name, description, readme });
|
| 277 |
+
const reply = await callLlm({ messages, model, signal: ctrl.signal });
|
| 278 |
+
if (reply == null) return null;
|
| 279 |
+
|
| 280 |
+
const obj = extractJsonObject(reply);
|
| 281 |
+
if (!obj || !Array.isArray(obj.categories)) {
|
| 282 |
+
console.warn(
|
| 283 |
+
`[categorize] ${spaceId}: malformed LLM reply (truncated): ` +
|
| 284 |
+
`${reply.slice(0, 120)}`,
|
| 285 |
+
);
|
| 286 |
+
return null;
|
| 287 |
+
}
|
| 288 |
+
return sanitizeSlugs(obj.categories, MAX_CATEGORIES_PER_APP);
|
| 289 |
+
} finally {
|
| 290 |
+
clearTimeout(timeoutId);
|
| 291 |
+
}
|
| 292 |
+
}
|
|
@@ -0,0 +1,290 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
/**
|
| 2 |
+
* Persistent cache for inferred app categories, backed by a
|
| 3 |
+
* HuggingFace dataset.
|
| 4 |
+
*
|
| 5 |
+
* Why a dataset (not a local file)
|
| 6 |
+
* ────────────────────────────────
|
| 7 |
+
* The website runs in a Docker HF Space. The container's
|
| 8 |
+
* filesystem is wiped on every rebuild (and rebuilds happen
|
| 9 |
+
* on every push, every model update, every Space restart).
|
| 10 |
+
* Re-running 200 LLM calls every cold start would be wasteful
|
| 11 |
+
* and slow the user-visible /api/js-apps for the first 30 s.
|
| 12 |
+
*
|
| 13 |
+
* Pushing the cache to a dataset gives us:
|
| 14 |
+
* 1. Persistence across rebuilds and machine moves
|
| 15 |
+
* 2. A versioned audit log of how categories evolve
|
| 16 |
+
* 3. A single source of truth other tooling can consume
|
| 17 |
+
* (the mobile shell could even read the dataset directly
|
| 18 |
+
* if it ever wanted to bypass the website).
|
| 19 |
+
*
|
| 20 |
+
* Storage shape
|
| 21 |
+
* ─────────────
|
| 22 |
+
* <dataset>/categories.json
|
| 23 |
+
*
|
| 24 |
+
* {
|
| 25 |
+
* "version": 1,
|
| 26 |
+
* "taxonomyVersion": 1,
|
| 27 |
+
* "updatedAt": "2026-05-10T11:08:42Z",
|
| 28 |
+
* "entries": {
|
| 29 |
+
* "<spaceId>": {
|
| 30 |
+
* "lastModified": "2026-05-08T22:13:01Z",
|
| 31 |
+
* "categories": ["storytelling", "kids", "voice"],
|
| 32 |
+
* "categorizedAt": "2026-05-10T11:08:42Z",
|
| 33 |
+
* "taxonomyVersion": 1
|
| 34 |
+
* }
|
| 35 |
+
* }
|
| 36 |
+
* }
|
| 37 |
+
*
|
| 38 |
+
* In-memory tier
|
| 39 |
+
* ──────────────
|
| 40 |
+
* The Map<spaceId, entry> is the hot path. The dataset is
|
| 41 |
+
* loaded once at boot and only flushed when entries actually
|
| 42 |
+
* change (the warmup batch buffers writes and flushes once
|
| 43 |
+
* at the end). All synchronous access goes through the Map.
|
| 44 |
+
*/
|
| 45 |
+
|
| 46 |
+
import { commit, createRepo } from '@huggingface/hub';
|
| 47 |
+
|
| 48 |
+
import { TAXONOMY_VERSION } from './categories.js';
|
| 49 |
+
|
| 50 |
+
// Default location: a per-user dataset that the HF_TOKEN owner
|
| 51 |
+
// definitely has write access to. Override with the env var
|
| 52 |
+
// when promoting to the org-owned `pollen-robotics/...` dataset.
|
| 53 |
+
const DEFAULT_DATASET = 'tfrere/reachy-mini-app-categories';
|
| 54 |
+
|
| 55 |
+
const CACHE_FILE_PATH = 'categories.json';
|
| 56 |
+
const CACHE_FORMAT_VERSION = 1;
|
| 57 |
+
|
| 58 |
+
class CategoryCache {
|
| 59 |
+
constructor() {
|
| 60 |
+
this.entries = new Map();
|
| 61 |
+
this.repoName = process.env.HF_CATEGORIES_DATASET || DEFAULT_DATASET;
|
| 62 |
+
this.loaded = false;
|
| 63 |
+
this.dirty = false;
|
| 64 |
+
// Concurrency guard for `flush()` - we never want two
|
| 65 |
+
// commit() calls fighting for the same parent commit.
|
| 66 |
+
this.flushing = false;
|
| 67 |
+
}
|
| 68 |
+
|
| 69 |
+
/**
|
| 70 |
+
* Load the dataset cache into memory. Best-effort: a missing
|
| 71 |
+
* dataset, a 404, or a malformed JSON all collapse to "start
|
| 72 |
+
* fresh, the warmup will repopulate". We never let cache load
|
| 73 |
+
* failure block the server boot.
|
| 74 |
+
*/
|
| 75 |
+
async load() {
|
| 76 |
+
if (this.loaded) return;
|
| 77 |
+
this.loaded = true;
|
| 78 |
+
|
| 79 |
+
const url = `https://huggingface.co/datasets/${this.repoName}/resolve/main/${CACHE_FILE_PATH}`;
|
| 80 |
+
try {
|
| 81 |
+
const res = await fetch(url, {
|
| 82 |
+
// Send the token even on a public dataset: it lets HF
|
| 83 |
+
// bump our rate limit and keeps the path identical for
|
| 84 |
+
// a future private dataset migration.
|
| 85 |
+
headers: process.env.HF_TOKEN
|
| 86 |
+
? { Authorization: `Bearer ${process.env.HF_TOKEN}` }
|
| 87 |
+
: undefined,
|
| 88 |
+
});
|
| 89 |
+
if (!res.ok) {
|
| 90 |
+
if (res.status === 404) {
|
| 91 |
+
console.log(
|
| 92 |
+
`[CategoryCache] Dataset ${this.repoName} or ${CACHE_FILE_PATH} ` +
|
| 93 |
+
`not found yet - starting empty.`,
|
| 94 |
+
);
|
| 95 |
+
} else {
|
| 96 |
+
console.warn(
|
| 97 |
+
`[CategoryCache] HTTP ${res.status} loading cache from ` +
|
| 98 |
+
`${this.repoName}, starting empty.`,
|
| 99 |
+
);
|
| 100 |
+
}
|
| 101 |
+
return;
|
| 102 |
+
}
|
| 103 |
+
const data = await res.json();
|
| 104 |
+
const entries = data?.entries || {};
|
| 105 |
+
let kept = 0;
|
| 106 |
+
let staleTaxonomy = 0;
|
| 107 |
+
for (const [id, raw] of Object.entries(entries)) {
|
| 108 |
+
if (!raw || typeof raw !== 'object') continue;
|
| 109 |
+
// Drop entries from a previous taxonomy: their slugs
|
| 110 |
+
// may no longer exist or may have shifted meaning.
|
| 111 |
+
// The warmup will re-run them.
|
| 112 |
+
if (raw.taxonomyVersion !== TAXONOMY_VERSION) {
|
| 113 |
+
staleTaxonomy++;
|
| 114 |
+
continue;
|
| 115 |
+
}
|
| 116 |
+
this.entries.set(id, {
|
| 117 |
+
lastModified: raw.lastModified || null,
|
| 118 |
+
categories: Array.isArray(raw.categories) ? raw.categories : [],
|
| 119 |
+
categorizedAt: raw.categorizedAt || null,
|
| 120 |
+
taxonomyVersion: raw.taxonomyVersion,
|
| 121 |
+
});
|
| 122 |
+
kept++;
|
| 123 |
+
}
|
| 124 |
+
console.log(
|
| 125 |
+
`[CategoryCache] Loaded ${kept} entries from ${this.repoName}` +
|
| 126 |
+
(staleTaxonomy ? ` (dropped ${staleTaxonomy} stale taxonomy)` : ''),
|
| 127 |
+
);
|
| 128 |
+
} catch (err) {
|
| 129 |
+
console.warn(
|
| 130 |
+
`[CategoryCache] Load failed (${err.message}); starting empty.`,
|
| 131 |
+
);
|
| 132 |
+
}
|
| 133 |
+
}
|
| 134 |
+
|
| 135 |
+
get(spaceId) {
|
| 136 |
+
return this.entries.get(spaceId) || null;
|
| 137 |
+
}
|
| 138 |
+
|
| 139 |
+
/**
|
| 140 |
+
* Decide whether `spaceId` needs a fresh classification call.
|
| 141 |
+
* It does when:
|
| 142 |
+
* - we have no entry at all, OR
|
| 143 |
+
* - the Space's `lastModified` has moved past our cached one
|
| 144 |
+
* (the README may have changed - re-classify), OR
|
| 145 |
+
* - the taxonomy version moved (handled at load() time, but
|
| 146 |
+
* belt-and-braces for hot reloads).
|
| 147 |
+
*/
|
| 148 |
+
needsCategorization(spaceId, lastModified) {
|
| 149 |
+
const entry = this.entries.get(spaceId);
|
| 150 |
+
if (!entry) return true;
|
| 151 |
+
if (entry.taxonomyVersion !== TAXONOMY_VERSION) return true;
|
| 152 |
+
if (lastModified && entry.lastModified !== lastModified) return true;
|
| 153 |
+
return false;
|
| 154 |
+
}
|
| 155 |
+
|
| 156 |
+
set(spaceId, { categories, lastModified }) {
|
| 157 |
+
if (!Array.isArray(categories)) return;
|
| 158 |
+
const next = {
|
| 159 |
+
lastModified: lastModified || null,
|
| 160 |
+
categories: [...categories],
|
| 161 |
+
categorizedAt: new Date().toISOString(),
|
| 162 |
+
taxonomyVersion: TAXONOMY_VERSION,
|
| 163 |
+
};
|
| 164 |
+
const prev = this.entries.get(spaceId);
|
| 165 |
+
// Skip the dirty flag if nothing actually changed - avoids
|
| 166 |
+
// a useless commit when a refresh confirms the same labels.
|
| 167 |
+
if (
|
| 168 |
+
prev &&
|
| 169 |
+
prev.lastModified === next.lastModified &&
|
| 170 |
+
prev.taxonomyVersion === next.taxonomyVersion &&
|
| 171 |
+
JSON.stringify(prev.categories) === JSON.stringify(next.categories)
|
| 172 |
+
) {
|
| 173 |
+
return;
|
| 174 |
+
}
|
| 175 |
+
this.entries.set(spaceId, next);
|
| 176 |
+
this.dirty = true;
|
| 177 |
+
}
|
| 178 |
+
|
| 179 |
+
/**
|
| 180 |
+
* Persist the in-memory cache to the dataset (one commit, one
|
| 181 |
+
* file). No-op if nothing has changed since the last flush.
|
| 182 |
+
*
|
| 183 |
+
* Auto-creates the dataset on first write if it doesn't exist
|
| 184 |
+
* yet (so a brand-new `HF_CATEGORIES_DATASET` value bootstraps
|
| 185 |
+
* cleanly without manual setup).
|
| 186 |
+
*/
|
| 187 |
+
async flush() {
|
| 188 |
+
if (!this.dirty || this.flushing) return;
|
| 189 |
+
if (!process.env.HF_TOKEN) {
|
| 190 |
+
console.warn('[CategoryCache] HF_TOKEN missing; skipping flush.');
|
| 191 |
+
return;
|
| 192 |
+
}
|
| 193 |
+
this.flushing = true;
|
| 194 |
+
try {
|
| 195 |
+
const payload = this.serialize();
|
| 196 |
+
const blob = new Blob([JSON.stringify(payload, null, 2)], {
|
| 197 |
+
type: 'application/json',
|
| 198 |
+
});
|
| 199 |
+
|
| 200 |
+
const repo = { type: 'dataset', name: this.repoName };
|
| 201 |
+
const credentials = { accessToken: process.env.HF_TOKEN };
|
| 202 |
+
|
| 203 |
+
// First attempt: plain commit. If the dataset doesn't
|
| 204 |
+
// exist yet, the SDK throws and we fall through to
|
| 205 |
+
// create-then-commit. We never assume the dataset exists
|
| 206 |
+
// - that lets a fresh deploy auto-bootstrap.
|
| 207 |
+
try {
|
| 208 |
+
await commit({
|
| 209 |
+
repo,
|
| 210 |
+
credentials,
|
| 211 |
+
title: `Update categories (${this.entries.size} apps)`,
|
| 212 |
+
operations: [
|
| 213 |
+
{
|
| 214 |
+
operation: 'addOrUpdate',
|
| 215 |
+
path: CACHE_FILE_PATH,
|
| 216 |
+
content: blob,
|
| 217 |
+
},
|
| 218 |
+
],
|
| 219 |
+
});
|
| 220 |
+
} catch (err) {
|
| 221 |
+
const msg = err?.message || '';
|
| 222 |
+
const looksMissing =
|
| 223 |
+
msg.includes('404') ||
|
| 224 |
+
msg.toLowerCase().includes('not found') ||
|
| 225 |
+
msg.toLowerCase().includes('does not exist');
|
| 226 |
+
if (!looksMissing) throw err;
|
| 227 |
+
console.log(
|
| 228 |
+
`[CategoryCache] Dataset ${this.repoName} missing - creating it.`,
|
| 229 |
+
);
|
| 230 |
+
await createRepo({
|
| 231 |
+
repo,
|
| 232 |
+
credentials,
|
| 233 |
+
private: false,
|
| 234 |
+
// Re-using the same blob so the initial commit ships
|
| 235 |
+
// the cache content (instead of an empty repo
|
| 236 |
+
// followed by a no-op commit).
|
| 237 |
+
files: [
|
| 238 |
+
{
|
| 239 |
+
path: CACHE_FILE_PATH,
|
| 240 |
+
content: await blob.arrayBuffer(),
|
| 241 |
+
},
|
| 242 |
+
],
|
| 243 |
+
});
|
| 244 |
+
}
|
| 245 |
+
|
| 246 |
+
this.dirty = false;
|
| 247 |
+
console.log(
|
| 248 |
+
`[CategoryCache] Flushed ${this.entries.size} entries to ${this.repoName}`,
|
| 249 |
+
);
|
| 250 |
+
} catch (err) {
|
| 251 |
+
// We deliberately swallow flush errors so a HF outage
|
| 252 |
+
// doesn't break the running server. The next set() will
|
| 253 |
+
// re-flag dirty=true and the next flush() will retry.
|
| 254 |
+
console.error(
|
| 255 |
+
`[CategoryCache] Flush failed: ${err?.message || err}`,
|
| 256 |
+
);
|
| 257 |
+
} finally {
|
| 258 |
+
this.flushing = false;
|
| 259 |
+
}
|
| 260 |
+
}
|
| 261 |
+
|
| 262 |
+
serialize() {
|
| 263 |
+
const entries = {};
|
| 264 |
+
for (const [id, entry] of this.entries) {
|
| 265 |
+
entries[id] = entry;
|
| 266 |
+
}
|
| 267 |
+
return {
|
| 268 |
+
version: CACHE_FORMAT_VERSION,
|
| 269 |
+
taxonomyVersion: TAXONOMY_VERSION,
|
| 270 |
+
updatedAt: new Date().toISOString(),
|
| 271 |
+
entries,
|
| 272 |
+
};
|
| 273 |
+
}
|
| 274 |
+
|
| 275 |
+
/**
|
| 276 |
+
* Diagnostic snapshot for /api/js-apps's `categorization`
|
| 277 |
+
* sub-payload. Lets the mobile shell decide whether to show
|
| 278 |
+
* "loading categories..." or to render the chips immediately.
|
| 279 |
+
*/
|
| 280 |
+
stats() {
|
| 281 |
+
return {
|
| 282 |
+
total: this.entries.size,
|
| 283 |
+
dataset: this.repoName,
|
| 284 |
+
taxonomyVersion: TAXONOMY_VERSION,
|
| 285 |
+
};
|
| 286 |
+
}
|
| 287 |
+
}
|
| 288 |
+
|
| 289 |
+
// Singleton: there's only one cache per server process.
|
| 290 |
+
export const categoryCache = new CategoryCache();
|
|
@@ -1,9 +1,42 @@
|
|
| 1 |
import express from 'express';
|
|
|
|
| 2 |
import path from 'path';
|
| 3 |
import { fileURLToPath } from 'url';
|
| 4 |
|
|
|
|
|
|
|
|
|
|
| 5 |
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
| 6 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7 |
const app = express();
|
| 8 |
const PORT = process.env.PORT || 7860;
|
| 9 |
|
|
@@ -14,6 +47,18 @@ const HF_SPACES_API = 'https://huggingface.co/api/spaces';
|
|
| 14 |
// Note: HF API doesn't support pagination with filter=, so we use a high limit
|
| 15 |
const HF_SPACES_LIMIT = 1000;
|
| 16 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 17 |
// In-memory cache
|
| 18 |
let appsCache = {
|
| 19 |
data: null,
|
|
@@ -183,6 +228,221 @@ app.get('/api/apps', async (req, res) => {
|
|
| 183 |
}
|
| 184 |
});
|
| 185 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 186 |
// OAuth config endpoint - expose public OAuth variables to the frontend
|
| 187 |
// (Docker Spaces don't auto-inject window.huggingface.variables like static Spaces)
|
| 188 |
app.get('/api/oauth-config', (req, res) => {
|
|
@@ -235,8 +495,29 @@ app.get('*', (req, res) => {
|
|
| 235 |
async function warmCache() {
|
| 236 |
console.log('[Startup] Pre-warming cache...');
|
| 237 |
try {
|
| 238 |
-
await getApps();
|
| 239 |
console.log('[Startup] Cache warmed successfully');
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 240 |
} catch (err) {
|
| 241 |
console.error('[Startup] Failed to warm cache:', err);
|
| 242 |
}
|
|
|
|
| 1 |
import express from 'express';
|
| 2 |
+
import { existsSync, readFileSync } from 'fs';
|
| 3 |
import path from 'path';
|
| 4 |
import { fileURLToPath } from 'url';
|
| 5 |
|
| 6 |
+
import { categorizeApp, HfTokenMissingError } from './categorize.js';
|
| 7 |
+
import { categoryCache } from './categoryCache.js';
|
| 8 |
+
|
| 9 |
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
| 10 |
|
| 11 |
+
// Load `.env` from the repo root in dev. In production (HF Space)
|
| 12 |
+
// the platform already injects the secrets as env vars, so this
|
| 13 |
+
// loader silently no-ops. We avoid the `dotenv` dep on purpose -
|
| 14 |
+
// the format is trivial, and reproducing it inline keeps the
|
| 15 |
+
// runtime closure tiny.
|
| 16 |
+
(function loadDotenv() {
|
| 17 |
+
try {
|
| 18 |
+
const envPath = path.join(__dirname, '..', '.env');
|
| 19 |
+
if (!existsSync(envPath)) return;
|
| 20 |
+
const text = readFileSync(envPath, 'utf8');
|
| 21 |
+
for (const line of text.split(/\r?\n/)) {
|
| 22 |
+
const m = line.match(/^\s*([A-Z0-9_]+)\s*=\s*(.*?)\s*$/i);
|
| 23 |
+
if (!m) continue;
|
| 24 |
+
const [, key, raw] = m;
|
| 25 |
+
let value = raw;
|
| 26 |
+
if (
|
| 27 |
+
(value.startsWith('"') && value.endsWith('"')) ||
|
| 28 |
+
(value.startsWith("'") && value.endsWith("'"))
|
| 29 |
+
) {
|
| 30 |
+
value = value.slice(1, -1);
|
| 31 |
+
}
|
| 32 |
+
// Existing env wins (so `HF_TOKEN=foo node …` overrides .env).
|
| 33 |
+
if (process.env[key] === undefined) process.env[key] = value;
|
| 34 |
+
}
|
| 35 |
+
} catch {
|
| 36 |
+
/* best-effort - missing or malformed .env never blocks boot */
|
| 37 |
+
}
|
| 38 |
+
})();
|
| 39 |
+
|
| 40 |
const app = express();
|
| 41 |
const PORT = process.env.PORT || 7860;
|
| 42 |
|
|
|
|
| 47 |
// Note: HF API doesn't support pagination with filter=, so we use a high limit
|
| 48 |
const HF_SPACES_LIMIT = 1000;
|
| 49 |
|
| 50 |
+
// Tag that gates the JS-only subset surfaced by /api/js-apps and
|
| 51 |
+
// fed to the LLM categorizer. Mirrors the filter the mobile shell
|
| 52 |
+
// applies today client-side; the route lets us retire that filter
|
| 53 |
+
// from the mobile codebase down the line.
|
| 54 |
+
const JS_APP_TAG = 'reachy_mini_js_app';
|
| 55 |
+
|
| 56 |
+
// Serialised LLM batch concurrency: we want at most one
|
| 57 |
+
// categorization sweep running at a time, regardless of how many
|
| 58 |
+
// /api/js-apps requests come in. The flag also prevents the
|
| 59 |
+
// startup warm-up and an on-demand refresh from racing each other.
|
| 60 |
+
let categorizationBatchRunning = false;
|
| 61 |
+
|
| 62 |
// In-memory cache
|
| 63 |
let appsCache = {
|
| 64 |
data: null,
|
|
|
|
| 228 |
}
|
| 229 |
});
|
| 230 |
|
| 231 |
+
// =====================================================================
|
| 232 |
+
// JS apps + LLM-inferred categories
|
| 233 |
+
// =====================================================================
|
| 234 |
+
//
|
| 235 |
+
// `/api/js-apps` is a curated view on top of `/api/apps`:
|
| 236 |
+
// 1. Filter on the `reachy_mini_js_app` tag (the mobile-embeddable subset).
|
| 237 |
+
// 2. Enrich each entry with `categories` + `categories_source`,
|
| 238 |
+
// sourced from a persistent dataset cache (see categoryCache.js).
|
| 239 |
+
//
|
| 240 |
+
// Categories are inferred lazily by an LLM from each Space's
|
| 241 |
+
// README. The first request after a cold start may see entries
|
| 242 |
+
// with `categories: null` while the warmup batch is still in
|
| 243 |
+
// flight; subsequent requests pick them up as the cache fills.
|
| 244 |
+
|
| 245 |
+
/**
|
| 246 |
+
* Pull the JS-app subset out of the global apps cache and fold
|
| 247 |
+
* in cached categories. Pure, synchronous-ish (the only async
|
| 248 |
+
* call is to the upstream `getApps()` which has its own cache).
|
| 249 |
+
*/
|
| 250 |
+
async function getJsApps() {
|
| 251 |
+
const apps = await getApps();
|
| 252 |
+
const jsApps = apps.filter((a) => {
|
| 253 |
+
const tags = a?.extra?.tags;
|
| 254 |
+
return Array.isArray(tags) && tags.includes(JS_APP_TAG);
|
| 255 |
+
});
|
| 256 |
+
|
| 257 |
+
return jsApps.map((app) => {
|
| 258 |
+
const cached = categoryCache.get(app.id);
|
| 259 |
+
return {
|
| 260 |
+
...app,
|
| 261 |
+
categories: cached ? cached.categories : null,
|
| 262 |
+
categories_source: cached ? 'inferred' : null,
|
| 263 |
+
categorized_at: cached ? cached.categorizedAt : null,
|
| 264 |
+
};
|
| 265 |
+
});
|
| 266 |
+
}
|
| 267 |
+
|
| 268 |
+
/**
|
| 269 |
+
* Run one classification pass over `jsApps`. Skips entries whose
|
| 270 |
+
* cache is still fresh (same `lastModified`, same taxonomy).
|
| 271 |
+
*
|
| 272 |
+
* Serial on purpose: HF Inference Providers don't love bursts
|
| 273 |
+
* from a single token, and total throughput on ~50 apps stays
|
| 274 |
+
* well under a minute. We slip a small jitter between calls to
|
| 275 |
+
* smooth the curve further.
|
| 276 |
+
*/
|
| 277 |
+
async function runCategorizationBatch(jsApps) {
|
| 278 |
+
if (categorizationBatchRunning) {
|
| 279 |
+
console.log('[Categorize] Batch already running, skipping.');
|
| 280 |
+
return;
|
| 281 |
+
}
|
| 282 |
+
if (!process.env.HF_TOKEN) {
|
| 283 |
+
console.warn(
|
| 284 |
+
'[Categorize] HF_TOKEN not set; skipping batch. Set it in .env ' +
|
| 285 |
+
'or the Space secrets to enable category inference.',
|
| 286 |
+
);
|
| 287 |
+
return;
|
| 288 |
+
}
|
| 289 |
+
|
| 290 |
+
const todo = jsApps.filter((app) =>
|
| 291 |
+
categoryCache.needsCategorization(app.id, app?.extra?.lastModified),
|
| 292 |
+
);
|
| 293 |
+
|
| 294 |
+
if (todo.length === 0) {
|
| 295 |
+
console.log(
|
| 296 |
+
`[Categorize] All ${jsApps.length} JS apps are already categorized.`,
|
| 297 |
+
);
|
| 298 |
+
return;
|
| 299 |
+
}
|
| 300 |
+
|
| 301 |
+
categorizationBatchRunning = true;
|
| 302 |
+
console.log(
|
| 303 |
+
`[Categorize] Starting batch: ${todo.length}/${jsApps.length} app(s) need classification.`,
|
| 304 |
+
);
|
| 305 |
+
|
| 306 |
+
let success = 0;
|
| 307 |
+
let failed = 0;
|
| 308 |
+
let aborted = false;
|
| 309 |
+
|
| 310 |
+
for (let i = 0; i < todo.length; i++) {
|
| 311 |
+
const app = todo[i];
|
| 312 |
+
const desc =
|
| 313 |
+
app.description ||
|
| 314 |
+
app.extra?.cardData?.short_description ||
|
| 315 |
+
'';
|
| 316 |
+
try {
|
| 317 |
+
const slugs = await categorizeApp({
|
| 318 |
+
spaceId: app.id,
|
| 319 |
+
name: app.name,
|
| 320 |
+
description: desc,
|
| 321 |
+
});
|
| 322 |
+
if (slugs == null) {
|
| 323 |
+
failed++;
|
| 324 |
+
console.log(
|
| 325 |
+
`[Categorize] (${i + 1}/${todo.length}) ${app.id}: transient failure, will retry next pass`,
|
| 326 |
+
);
|
| 327 |
+
} else {
|
| 328 |
+
categoryCache.set(app.id, {
|
| 329 |
+
categories: slugs,
|
| 330 |
+
lastModified: app.extra?.lastModified || null,
|
| 331 |
+
});
|
| 332 |
+
success++;
|
| 333 |
+
console.log(
|
| 334 |
+
`[Categorize] (${i + 1}/${todo.length}) ${app.id}: ${
|
| 335 |
+
slugs.length ? slugs.join(', ') : '(no fit)'
|
| 336 |
+
}`,
|
| 337 |
+
);
|
| 338 |
+
}
|
| 339 |
+
} catch (err) {
|
| 340 |
+
if (err instanceof HfTokenMissingError) {
|
| 341 |
+
console.warn(
|
| 342 |
+
'[Categorize] HF_TOKEN missing mid-batch; aborting cleanly.',
|
| 343 |
+
);
|
| 344 |
+
aborted = true;
|
| 345 |
+
break;
|
| 346 |
+
}
|
| 347 |
+
failed++;
|
| 348 |
+
console.warn(
|
| 349 |
+
`[Categorize] (${i + 1}/${todo.length}) ${app.id}: error - ${err.message}`,
|
| 350 |
+
);
|
| 351 |
+
}
|
| 352 |
+
|
| 353 |
+
// 250 ms cooldown between calls. Below this, the HF Provider
|
| 354 |
+
// router occasionally rate-limits a hot token.
|
| 355 |
+
await new Promise((resolve) => setTimeout(resolve, 250));
|
| 356 |
+
}
|
| 357 |
+
|
| 358 |
+
console.log(
|
| 359 |
+
`[Categorize] Batch done: ${success} ok, ${failed} failed${aborted ? ' (aborted)' : ''}.`,
|
| 360 |
+
);
|
| 361 |
+
// Persist the new entries even if some failed - partial
|
| 362 |
+
// progress is strictly better than none, and the failed
|
| 363 |
+
// entries will be retried on the next pass.
|
| 364 |
+
await categoryCache.flush();
|
| 365 |
+
|
| 366 |
+
categorizationBatchRunning = false;
|
| 367 |
+
}
|
| 368 |
+
|
| 369 |
+
/**
|
| 370 |
+
* Wrap the diagnostic snapshot for the API payload. Lets
|
| 371 |
+
* consumers (mobile shell, website) decide whether to show
|
| 372 |
+
* "loading categories..." or render chips immediately.
|
| 373 |
+
*/
|
| 374 |
+
function buildCategorizationStats(jsApps) {
|
| 375 |
+
let withCategories = 0;
|
| 376 |
+
for (const app of jsApps) {
|
| 377 |
+
if (app.categories && app.categories.length >= 0 && app.categories_source) {
|
| 378 |
+
withCategories++;
|
| 379 |
+
}
|
| 380 |
+
}
|
| 381 |
+
return {
|
| 382 |
+
enabled: !!process.env.HF_TOKEN,
|
| 383 |
+
total: jsApps.length,
|
| 384 |
+
classified: withCategories,
|
| 385 |
+
pending: jsApps.length - withCategories,
|
| 386 |
+
inProgress: categorizationBatchRunning,
|
| 387 |
+
...categoryCache.stats(),
|
| 388 |
+
};
|
| 389 |
+
}
|
| 390 |
+
|
| 391 |
+
app.get('/api/js-apps', async (req, res) => {
|
| 392 |
+
try {
|
| 393 |
+
const apps = await getJsApps();
|
| 394 |
+
|
| 395 |
+
// Background top-up: if any entry is still uncategorized
|
| 396 |
+
// (or a Space's lastModified moved since we last looked),
|
| 397 |
+
// fire off a batch. We DO NOT await it - the response goes
|
| 398 |
+
// out immediately with whatever the cache currently knows.
|
| 399 |
+
const needsWork = apps.some(
|
| 400 |
+
(a) =>
|
| 401 |
+
!a.categories_source ||
|
| 402 |
+
categoryCache.needsCategorization(a.id, a.extra?.lastModified),
|
| 403 |
+
);
|
| 404 |
+
if (needsWork) {
|
| 405 |
+
// `void` to make it crystal clear we don't expect a value;
|
| 406 |
+
// the batch logs its own progress.
|
| 407 |
+
void runCategorizationBatch(apps).catch((err) => {
|
| 408 |
+
console.error('[Categorize] Background batch crashed:', err);
|
| 409 |
+
});
|
| 410 |
+
}
|
| 411 |
+
|
| 412 |
+
res.json({
|
| 413 |
+
apps,
|
| 414 |
+
cached: true,
|
| 415 |
+
cacheAge: appsCache.lastFetch
|
| 416 |
+
? Math.round((Date.now() - appsCache.lastFetch) / 1000)
|
| 417 |
+
: 0,
|
| 418 |
+
count: apps.length,
|
| 419 |
+
categorization: buildCategorizationStats(apps),
|
| 420 |
+
});
|
| 421 |
+
} catch (err) {
|
| 422 |
+
console.error('[API] /api/js-apps error:', err);
|
| 423 |
+
res.status(500).json({ error: 'Failed to fetch JS apps' });
|
| 424 |
+
}
|
| 425 |
+
});
|
| 426 |
+
|
| 427 |
+
// Manual trigger for a categorization sweep, useful when
|
| 428 |
+
// hand-tuning the taxonomy or testing the LLM prompt without
|
| 429 |
+
// waiting for the next /api/js-apps hit.
|
| 430 |
+
app.post('/api/js-apps/refresh-categories', async (req, res) => {
|
| 431 |
+
try {
|
| 432 |
+
const apps = await getJsApps();
|
| 433 |
+
void runCategorizationBatch(apps).catch((err) => {
|
| 434 |
+
console.error('[Categorize] Manual batch crashed:', err);
|
| 435 |
+
});
|
| 436 |
+
res.json({
|
| 437 |
+
ok: true,
|
| 438 |
+
message: `Categorization batch kicked off for ${apps.length} JS apps.`,
|
| 439 |
+
stats: buildCategorizationStats(apps),
|
| 440 |
+
});
|
| 441 |
+
} catch (err) {
|
| 442 |
+
res.status(500).json({ error: 'Failed to trigger refresh' });
|
| 443 |
+
}
|
| 444 |
+
});
|
| 445 |
+
|
| 446 |
// OAuth config endpoint - expose public OAuth variables to the frontend
|
| 447 |
// (Docker Spaces don't auto-inject window.huggingface.variables like static Spaces)
|
| 448 |
app.get('/api/oauth-config', (req, res) => {
|
|
|
|
| 495 |
async function warmCache() {
|
| 496 |
console.log('[Startup] Pre-warming cache...');
|
| 497 |
try {
|
| 498 |
+
const apps = await getApps();
|
| 499 |
console.log('[Startup] Cache warmed successfully');
|
| 500 |
+
|
| 501 |
+
// Categorization warm-up: fire the JS-app batch in the
|
| 502 |
+
// background so the first /api/js-apps caller doesn't
|
| 503 |
+
// shoulder the cold-start cost. Order: load the dataset
|
| 504 |
+
// cache first (cheap, one HTTP call), then run the batch
|
| 505 |
+
// for stale entries only.
|
| 506 |
+
void (async () => {
|
| 507 |
+
try {
|
| 508 |
+
await categoryCache.load();
|
| 509 |
+
const jsApps = apps.filter((a) => {
|
| 510 |
+
const tags = a?.extra?.tags;
|
| 511 |
+
return Array.isArray(tags) && tags.includes(JS_APP_TAG);
|
| 512 |
+
});
|
| 513 |
+
console.log(
|
| 514 |
+
`[Startup] Found ${jsApps.length} JS apps; checking categories...`,
|
| 515 |
+
);
|
| 516 |
+
await runCategorizationBatch(jsApps);
|
| 517 |
+
} catch (err) {
|
| 518 |
+
console.error('[Startup] Categorization warm-up failed:', err);
|
| 519 |
+
}
|
| 520 |
+
})();
|
| 521 |
} catch (err) {
|
| 522 |
console.error('[Startup] Failed to warm cache:', err);
|
| 523 |
}
|