// S795 — SemanticSearch.ts: retrieval semantico con embedding pre-calcolati (P31-RI) // // ARCHITETTURA: // Dev-time: scripts/gen-repo-embeddings.mjs genera public/repo-embeddings.json // usando @xenova/transformers + all-MiniLM-L6-v2 (Node.js). // Runtime: SemanticSearch.ts carica il JSON statico + embeds la query con // Transformers.js (WebAssembly, browser-native, ~23MB download cached IDB). // Fallback: Jaccard se embedding non disponibile o timeout. // // Integrazione: esportare `semanticGetRelevantFiles` come drop-in per `GetRelevantFilesFn` // in relevantFilesLoader.ts e agentLoop.ts. import type { RelevantFileHit } from "./relevantFilesLoader"; /** Struttura di un'entry nel JSON pre-calcolato */ export interface RepoEmbeddingEntry { /** path relativo alla root del repo */ path: string; /** descrizione sintetica del file (da commento header) */ description: string; /** keywords per fallback Jaccard */ keywords: string[]; /** embedding float32 all-MiniLM-L6-v2 (dim=384), omesso se not computed */ embedding?: number[]; } // ── Cache in-memory ──────────────────────────────────────────────────────────── let _entries: RepoEmbeddingEntry[] | null = null; let _embedderP: Promise<((text: string) => Promise) | null> | null = null; // ── Carica embeddings JSON dal bundle statico ───────────────────────────────── const EMBEDDINGS_URL = "/repo-embeddings.json"; async function _loadEntries(): Promise { if (_entries) return _entries; try { const r = await fetch(EMBEDDINGS_URL, { cache: "force-cache" }); if (!r.ok) throw new Error(`HTTP ${r.status}`); _entries = await r.json() as RepoEmbeddingEntry[]; return _entries; } catch (e) { console.warn("[SemanticSearch] repo-embeddings.json non disponibile:", e); _entries = []; return []; } } // ── Carica l'embedder Transformers.js (lazy, singleton) ────────────────────── async function _getEmbedder(): Promise<((text: string) => Promise) | null> { if (_embedderP) return _embedderP; _embedderP = (async () => { try { // @xenova/transformers funziona nel browser via WebAssembly (ONNX Runtime). // Il modello all-MiniLM-L6-v2 (quantized) è ~23MB, cached in IndexedDB dopo il primo download. // @ts-ignore — CDN URL import, no type declarations available // eslint-disable-next-line @typescript-eslint/ban-ts-comment const { pipeline } = await import(/* webpackIgnore: true */ "https://cdn.jsdelivr.net/npm/@xenova/transformers@2.17.2/dist/transformers.min.js" as any) as any; const extractor = await pipeline("feature-extraction", "Xenova/all-MiniLM-L6-v2", { quantized: true, progress_callback: undefined, }); return async (text: string): Promise => { const out = await extractor(text, { pooling: "mean", normalize: true }); return Array.from(out.data as Float32Array); }; } catch (e) { console.warn("[SemanticSearch] Transformers.js non disponibile:", e); return null; } })(); return _embedderP; } // ── Cosine similarity ───────────────────────────────────────────────────────── function _cosine(a: number[], b: number[]): number { let dot = 0, na = 0, nb = 0; const len = Math.min(a.length, b.length); for (let i = 0; i < len; i++) { dot += a[i] * b[i]; na += a[i] ** 2; nb += b[i] ** 2; } if (na === 0 || nb === 0) return 0; return dot / (Math.sqrt(na) * Math.sqrt(nb)); } // ── Fallback Jaccard (keyword overlap) ─────────────────────────────────────── function _jaccard(query: string, entry: RepoEmbeddingEntry): number { const qTokens = new Set(query.toLowerCase().split(/\W+/).filter(t => t.length > 2)); if (qTokens.size === 0) return 0; const eTokens = new Set([ ...entry.keywords, ...entry.path.toLowerCase().split(/[/._-]+/), ...entry.description.toLowerCase().split(/\W+/).filter(t => t.length > 2), ]); let inter = 0; qTokens.forEach(t => { if (eTokens.has(t)) inter++; }); return inter / (qTokens.size + eTokens.size - inter); } // ── API pubblica ────────────────────────────────────────────────────────────── /** * Cerca i file più semanticamente rilevanti alla query. * Drop-in per `GetRelevantFilesFn` di relevantFilesLoader.ts. * * @param query testo della query utente * @param topK numero max di risultati (default 6) * @param minScore soglia minima di score (default 0.15) * @param timeoutMs timeout per embedding (default 3000ms, poi fallback Jaccard) */ export async function semanticGetRelevantFiles( query: string, topK = 6, minScore = 0.15, timeoutMs = 3000, ): Promise { const entries = await _loadEntries(); if (entries.length === 0) return []; // Controlla se ci sono entries con embedding const hasEmbeddings = entries.some(e => e.embedding && e.embedding.length > 0); let scores: Array<{ entry: RepoEmbeddingEntry; score: number }>; if (hasEmbeddings) { // Tenta embedding della query con timeout const embedder = await Promise.race([ _getEmbedder(), new Promise(r => setTimeout(() => r(null), timeoutMs)), ]); if (embedder) { // Embedding semantico const qEmb = await embedder(query); scores = entries .filter(e => e.embedding && e.embedding.length > 0) .map(e => ({ entry: e, score: _cosine(qEmb, e.embedding!) })); } else { // Timeout → fallback Jaccard sull'embedding delle keywords console.info("[SemanticSearch] timeout embedder → fallback Jaccard"); scores = entries.map(e => ({ entry: e, score: _jaccard(query, e) })); } } else { // JSON senza embedding → Jaccard puro (modalità keyword-only) scores = entries.map(e => ({ entry: e, score: _jaccard(query, e) })); } return scores .filter(s => s.score >= minScore) .sort((a, b) => b.score - a.score) .slice(0, topK) .map(s => ({ path: s.entry.path, content: s.entry.description, // placeholder — il contenuto vero arriva da VFS score: s.score, })); } /** * Precaricare l'embedder in background (chiama subito dopo il mount dell'app * per non pagare il cold-start durante la prima query utente). */ export function warmupEmbedder(): void { _getEmbedder().catch(() => { /* non-blocking */ }); } /** * Stato del sistema embedding per diagnostica. */ export function getSemanticSearchStatus(): { entriesLoaded: number; hasEmbeddings: boolean; embedderReady: boolean; } { return { entriesLoaded: _entries?.length ?? 0, hasEmbeddings: !!(_entries?.some(e => e.embedding && e.embedding.length > 0)), embedderReady: _embedderP !== null, }; }