import { useState, useEffect, useCallback, useRef } from 'react';
import type { Document, Chunk, EmbeddedChunk, ModelState } from './types';
import { loadAllModels, isAllModelsReady } from './pipeline/models';
import { chunkDocument, extractTitle } from './pipeline/chunking';
import { embedDocChunksBatch } from './pipeline/embeddings';
import { BM25Index } from './pipeline/bm25';
import { runPipeline } from './pipeline/orchestrator';
import type { PipelineState } from './components/PipelineView';
import QueryInput from './components/QueryInput';
import ModelStatus from './components/ModelStatus';
import PipelineView from './components/PipelineView';
import DocumentManager from './components/DocumentManager';
const SAMPLE_DOCS = [
'api-design-principles.md',
'distributed-systems-overview.md',
'machine-learning-primer.md',
'history-of-coffee.md',
];
const SHOWCASE_CARDS = [
{
title: 'Faithful to qmd',
body: 'BM25, vector search, query expansion, RRF fusion, and reranking follow the upstream retrieval recipe instead of flattening everything into one model call.',
},
{
title: 'Browser-native bits',
body: 'Transformers.js and WebGPU run the pipeline locally, cache model weights in the browser, and expose each stage so the search system stays inspectable.',
},
];
const INDEX_BATCH_SIZE = 8;
const INITIAL_PIPELINE: PipelineState = {
expansion: { status: 'idle' },
search: { status: 'idle' },
rrf: { status: 'idle' },
rerank: { status: 'idle' },
blend: { status: 'idle' },
};
function upsertDocuments(current: Document[], incoming: Document[]): Document[] {
const merged = new Map(current.map((doc) => [doc.id, doc]));
for (const doc of incoming) {
merged.set(doc.id, doc);
}
return [...merged.values()];
}
function ShowcaseCard({ title, body }: { title: string; body: string }) {
return (
);
}
function App() {
const [models, setModels] = useState([
{ name: 'embedding', status: 'pending', progress: 0 },
{ name: 'reranker', status: 'pending', progress: 0 },
{ name: 'expansion', status: 'pending', progress: 0 },
]);
const [documents, setDocuments] = useState([]);
const [chunks, setChunks] = useState([]);
const [embeddedChunks, setEmbeddedChunks] = useState([]);
const [bm25Index, setBm25Index] = useState(null);
const [pipeline, setPipeline] = useState(INITIAL_PIPELINE);
const [indexing, setIndexing] = useState(false);
const [indexingProgress, setIndexingProgress] = useState({ completed: 0, total: 0 });
const [query, setQuery] = useState('');
const [intent, setIntent] = useState();
const [dark, setDark] = useState(() =>
document.documentElement.getAttribute('data-theme') === 'dark',
);
const searchRunIdRef = useRef(0);
const embeddingReady = models.find((model) => model.name === 'embedding')?.status === 'ready';
useEffect(() => {
loadAllModels((state) => {
setModels((prev) => prev.map((model) => (
model.name === state.name ? state : model
)));
}).catch(console.error);
}, []);
useEffect(() => {
async function loadSampleDocs() {
try {
const loadedDocs = await Promise.all(
SAMPLE_DOCS.map(async (filename) => {
const response = await fetch(`/eval-docs/${filename}`);
const body = await response.text();
const title = extractTitle(body, filename);
return { id: filename, title, body, filepath: filename };
}),
);
setDocuments((prev) => upsertDocuments(prev, loadedDocs));
} catch (error) {
console.error(error);
}
}
loadSampleDocs();
}, []);
useEffect(() => {
if (documents.length === 0) {
setChunks([]);
setEmbeddedChunks([]);
setBm25Index(null);
setIndexing(false);
setIndexingProgress({ completed: 0, total: 0 });
return;
}
const nextChunks = documents.flatMap((doc) => chunkDocument(doc));
setChunks(nextChunks);
setBm25Index(new BM25Index(nextChunks));
}, [documents]);
useEffect(() => {
let cancelled = false;
if (!embeddingReady || chunks.length === 0) {
setEmbeddedChunks([]);
setIndexing(false);
setIndexingProgress({ completed: 0, total: chunks.length });
return () => {
cancelled = true;
};
}
async function embedChunks() {
setIndexing(true);
setIndexingProgress({ completed: 0, total: chunks.length });
const embedded: EmbeddedChunk[] = [];
for (let i = 0; i < chunks.length; i += INDEX_BATCH_SIZE) {
const batch = chunks.slice(i, i + INDEX_BATCH_SIZE);
const embeddings = await embedDocChunksBatch(
batch.map((chunk) => ({ title: chunk.title, text: chunk.text })),
);
if (cancelled) return;
for (let j = 0; j < batch.length; j++) {
const chunk = batch[j];
const embedding = embeddings[j];
if (!chunk || !embedding) continue;
embedded.push({ ...chunk, embedding });
}
setIndexingProgress({
completed: Math.min(i + batch.length, chunks.length),
total: chunks.length,
});
}
if (cancelled) return;
setEmbeddedChunks(embedded);
setIndexing(false);
}
embedChunks().catch((error) => {
if (cancelled) return;
console.error(error);
setEmbeddedChunks([]);
setIndexing(false);
});
return () => {
cancelled = true;
};
}, [chunks, embeddingReady]);
const handleUpload = useCallback(async (files: FileList) => {
const uploadedDocs = await Promise.all(
Array.from(files).map(async (file) => {
const body = await file.text();
const title = extractTitle(body, file.name);
return { id: file.name, title, body, filepath: file.name };
}),
);
setDocuments((prev) => upsertDocuments(prev, uploadedDocs));
}, []);
const handlePaste = useCallback((text: string, filename: string) => {
const title = extractTitle(text, filename);
setDocuments((prev) => upsertDocuments(prev, [
{ id: filename, title, body: text, filepath: filename },
]));
}, []);
const handleSearch = useCallback(async (searchQuery: string, searchIntent?: string) => {
if (!bm25Index || embeddedChunks.length === 0) return;
const runId = ++searchRunIdRef.current;
setQuery(searchQuery);
setIntent(searchIntent);
setPipeline(INITIAL_PIPELINE);
const generator = runPipeline({
query: searchQuery,
intent: searchIntent,
embeddedChunks,
bm25Index,
});
for await (const event of generator) {
if (searchRunIdRef.current !== runId) return;
setPipeline((prev) => ({
...prev,
[event.stage]: {
status: event.status,
...('data' in event ? { data: event.data } : {}),
...('error' in event ? { error: event.error } : {}),
},
}));
}
}, [bm25Index, embeddedChunks]);
const allReady = isAllModelsReady() && embeddedChunks.length > 0 && !indexing;
const toggleTheme = useCallback(() => {
setDark((prev) => {
const next = !prev;
document.documentElement.setAttribute('data-theme', next ? 'dark' : 'light');
localStorage.setItem('qmd-theme', next ? 'dark' : 'light');
return next;
});
}, []);
return (
{indexing && (
Indexing local chunks in the browser ({indexingProgress.completed}/{indexingProgress.total})...
)}
{query &&
}
({ id: doc.id, title: doc.title, filepath: doc.filepath }))}
onUpload={handleUpload}
onPaste={handlePaste}
/>
);
}
export default App;