/** * Content moderation for JS Reachy Mini apps surfaced in the mobile * catalog. * * Why this exists * ─────────────── * The mobile shell embeds third-party Hugging Face Spaces in a * WebView iframe. Apple App Store guideline 1.2 (and the Google Play * UGC policy) require the host to *filter objectionable material* * before it reaches users. This module is the automated half of that * obligation; the manual half is the hand-edited `blocked-app-list.json` * killswitch on the official dataset (see `index.js`). * * Two-layer pipeline (`moderateApp`) * ────────────────────────────────── * 1. REGEX prescreen (synchronous, free): a tiny list of patterns * that are objectionable 100% of the time regardless of context * (explicit sexual content, CSAM signals, obvious scams). A hit * is a hard block and short-circuits the LLM call. * 2. LLM classifier (HF Inference Providers, ~1 s, cached): for * everything the regex doesn't catch, an 8B model returns a * STRUCTURED verdict against a CLOSED policy taxonomy. We never * ask it for a free-form judgment or a fuzzy score - the closed * list keeps the output auditable (each block has a category + * reason a reviewer can be shown) and stable. * * Three outcomes, not two * ─────────────────────── * The verdict `decision` is one of `allow` | `block` | `review`. * The catalog runs fail-closed (see `computeVisibility` in `index.js`): * ONLY an explicit `allow` is shown. `block` (clear violation) and * `review` (the LLM is unsure) are both quarantined. The `review` * bucket therefore over-blocks-on-doubt by design - an ambiguous app * stays hidden until a human triages it (e.g. via the manual * allow/block lists) rather than being exposed to users. * * Robustness contract * ─────────────────── * `moderateApp` NEVER throws on transient failure (network, 429, * malformed JSON). It returns `null`, which the cache layer reads as * "not yet moderated; retry next pass". Because the catalog is * fail-closed, an unmoderated app stays HIDDEN until a verdict lands, * so an upstream hiccup never leaks unmoderated content (it just keeps * the app out of the catalog a little longer). Hard errors (HF_TOKEN * missing) throw `HfTokenMissingError` so the caller can short-circuit * the batch. * * This mirrors `categorize.js` on purpose: same HF Inference path, * same README fetch/clean (imported, not duplicated), same JSON * extraction, same caching shape. Keep the two in sync when one * evolves. */ import { cleanReadme, fetchSpaceReadme, HfTokenMissingError } from './categorize.js'; export { HfTokenMissingError }; const HF_INFERENCE_URL = 'https://router.huggingface.co/v1/chat/completions'; const DEFAULT_MODEL = 'meta-llama/Llama-3.1-8B-Instruct'; const README_MAX_CHARS = 3000; const LLM_TIMEOUT_MS = 30_000; const LLM_MAX_TOKENS = 120; const LLM_TEMPERATURE = 0; // Bump when the regex list or the LLM prompt changes in a way that // should re-moderate the whole catalog. The cache drops entries from // an older policy version on load (see `moderationCache.js`), exactly // like the taxonomy version gate in the category cache. export const MODERATION_POLICY_VERSION = 1; // Closed policy taxonomy. The LLM MUST pick exactly one. `none` is // the "clean" outcome. Anything else maps to a block/review action in // `decisionForCategory()`. export const POLICY_CATEGORIES = [ 'sexual', // pornographic / explicit sexual content 'hate', // hate speech, harassment, slurs targeting a protected group 'violence', // graphic violence, gore, glorification of violence 'illegal', // illegal goods/acts, weapons, drugs marketplace 'scam_malware', // phishing, wallet drainers, malware, deceptive money grabs 'self_harm', // promotion of self-harm / suicide / eating disorders 'none', // nothing objectionable ]; const HARD_CATEGORIES = new Set(['sexual', 'hate', 'illegal', 'self_harm']); // ===================================================================== // Layer 1 - regex prescreen // ===================================================================== // // KEEP THIS LIST SMALL AND UNAMBIGUOUS. Only patterns that are // objectionable in EVERY context belong here - the regex layer has no // notion of nuance, so any ambiguous word ("kill", "shoot", "drug") // would generate false positives on perfectly fine apps (games, dev // tools). Nuance is the LLM's job (layer 2). The team should extend // the hate-term list from a maintained lexicon rather than inline. const HARD_PATTERNS = [ { category: 'sexual', re: /\b(porn(?:hub)?|xxx+|hentai|camgirl|onlyfans|nudify|deepnude|sexcam|escort\s+service)\b/i, }, { // CSAM signals: age-of-minor token co-occurring with explicit // sexual token. Deliberately narrow to avoid flagging "kids" // apps; a match here is always a hard block. category: 'illegal', re: /\b(child|underage|minor|preteen|loli(?:con)?)\b[\s\S]{0,40}\b(porn|nude|sex|nsfw)\b/i, }, { category: 'scam_malware', re: /\b(free\s+(?:robux|v-?bucks|bitcoin|crypto|gift\s*cards?)|seed\s+phrase|wallet\s+drainer|private\s+key\s+generator)\b/i, }, { category: 'violence', re: /\b(gore|beheading|snuff\s+film|execution\s+video)\b/i, }, ]; /** * Run the regex prescreen against the app's text. Returns a hard-block * verdict on the first hit, or `null` when nothing matches (the caller * then falls through to the LLM). * * Synchronous and free - safe to run on every app before deciding * whether an LLM call is even needed. */ export function regexPrescreen({ name, description, readme }) { const haystack = `${name || ''}\n${description || ''}\n${readme || ''}`; for (const { category, re } of HARD_PATTERNS) { if (re.test(haystack)) { return { decision: 'block', category, reason: `regex hard-block: matched ${category} pattern`, severity: 'hard', source: 'regex', policyVersion: MODERATION_POLICY_VERSION, }; } } return null; } /** * Map a closed-taxonomy category to a default action when the LLM * returns a category but an unexpected/absent `decision`. Hard * categories block; the softer ones (violence, scam_malware) also * block but at `soft` severity; `none` allows. */ function decisionForCategory(category) { if (category === 'none') return { decision: 'allow', severity: null }; if (HARD_CATEGORIES.has(category)) return { decision: 'block', severity: 'hard' }; return { decision: 'block', severity: 'soft' }; } // ===================================================================== // Layer 2 - LLM classifier // ===================================================================== function buildMessages({ name, description, readme }) { const categories = POLICY_CATEGORIES.join(', '); const system = `You are a content-safety classifier for a GENERAL-AUDIENCE (12+) catalog of small robot apps. Each app is a web page embedded in a companion app for a desk robot. OUTPUT FORMAT Return ONLY a single JSON object, no prose, no code fences: {"decision": "allow"|"block"|"review", "category": "", "reason": ""} CATEGORY (pick EXACTLY ONE slug from this closed list) ${categories} DECISION RULES - "block": the app clearly contains or promotes objectionable material: pornographic/explicit sexual content (sexual), hate speech or harassment (hate), graphic violence/gore (violence), illegal goods/acts incl. anything sexualizing minors (illegal), phishing/malware/deceptive money grabs (scam_malware), promotion of self-harm or suicide (self_harm). - "allow": ordinary robot apps - games, music, dancing, storytelling, companions, voice assistants, vision demos, dev tools, education. Edgy-but-harmless humor is allowed. Use category "none". - "review": you genuinely cannot tell from the text whether it is appropriate (ambiguous, too little signal, mixed). Pick the most likely category and let a human decide. IMPORTANT - Do NOT block an app just for being technical, weird, or low-quality. - "kids", "children", "bedtime" framing is a NORMAL audience, not a red flag, UNLESS combined with sexual/abusive content. - Judge the app's PURPOSE, not isolated words. Return the JSON now.`; const user = `App name: ${name || '(unknown)'}\n` + `Short description: ${description || '(none)'}\n\n` + `README excerpt:\n${readme || '(no README available)'}\n\n` + 'Classify it.'; return [ { role: 'system', content: system }, { role: 'user', content: user }, ]; } /** * Best-effort JSON extraction - grabs the first balanced `{...}` * block and parses it. Mirrors the extractor in `categorize.js` * because some 8B providers still wrap the answer in fences. */ function extractJsonObject(text) { if (!text || typeof text !== 'string') return null; const start = text.indexOf('{'); if (start === -1) return null; let depth = 0; for (let i = start; i < text.length; i++) { const ch = text[i]; if (ch === '{') depth++; else if (ch === '}') { depth--; if (depth === 0) { try { return JSON.parse(text.slice(start, i + 1)); } catch { return null; } } } } return null; } async function callLlm({ messages, model, signal }) { const token = process.env.HF_TOKEN; if (!token) throw new HfTokenMissingError(); const body = { model, messages, temperature: LLM_TEMPERATURE, max_tokens: LLM_MAX_TOKENS, response_format: { type: 'json_object' }, }; let res; try { res = await fetch(HF_INFERENCE_URL, { method: 'POST', headers: { Authorization: `Bearer ${token}`, 'Content-Type': 'application/json', }, body: JSON.stringify(body), signal, }); } catch (err) { console.warn(`[moderate] LLM fetch failed: ${err.message}`); return null; } if (!res.ok) { const detail = await res.text().catch(() => ''); console.warn(`[moderate] LLM HTTP ${res.status}: ${detail.slice(0, 200)}`); return null; } let json; try { json = await res.json(); } catch { return null; } return json?.choices?.[0]?.message?.content ?? null; } /** * Normalize a raw LLM object into a validated verdict, or `null` if * it's unusable (so the caller treats it as a transient miss). */ function normalizeVerdict(obj) { if (!obj || typeof obj !== 'object') return null; let category = typeof obj.category === 'string' ? obj.category.trim() : ''; if (!POLICY_CATEGORIES.includes(category)) category = ''; let decision = typeof obj.decision === 'string' ? obj.decision.trim() : ''; if (!['allow', 'block', 'review'].includes(decision)) decision = ''; // If the model gave a category but no usable decision, derive it. if (!decision && category) { decision = decisionForCategory(category).decision; } if (!decision) return null; if (!category) category = decision === 'allow' ? 'none' : 'none'; const severity = decision === 'block' ? decisionForCategory(category).severity || 'soft' : null; const reason = typeof obj.reason === 'string' && obj.reason.trim() ? obj.reason.trim().slice(0, 200) : `llm: ${decision}/${category}`; return { decision, category, reason, severity, source: 'llm', policyVersion: MODERATION_POLICY_VERSION, }; } /** * Public entry point. Returns a verdict object: * { decision, category, reason, severity, source, policyVersion } * or `null` on transient failure (retry next pass). * * Official apps should be skipped by the caller (they are curated by * Pollen and don't need moderating) - this keeps LLM load down and * avoids false positives on first-party content. */ export async function moderateApp({ name, description, spaceId, model = DEFAULT_MODEL, } = {}) { if (!spaceId) return null; const ctrl = new AbortController(); const timeoutId = setTimeout(() => ctrl.abort(), LLM_TIMEOUT_MS); try { const rawReadme = await fetchSpaceReadme(spaceId, { signal: ctrl.signal }); const readme = cleanReadme(rawReadme).slice(0, README_MAX_CHARS); // Layer 1: free, deterministic. A hit short-circuits the LLM. const regexVerdict = regexPrescreen({ name, description, readme }); if (regexVerdict) return regexVerdict; // Layer 2: LLM nuance. const messages = buildMessages({ name, description, readme }); const reply = await callLlm({ messages, model, signal: ctrl.signal }); if (reply == null) return null; const verdict = normalizeVerdict(extractJsonObject(reply)); if (!verdict) { console.warn( `[moderate] ${spaceId}: malformed LLM reply (truncated): ${reply.slice(0, 120)}`, ); return null; } return verdict; } finally { clearTimeout(timeoutId); } }