add pipeline orchestrator as AsyncGenerator yielding stage events

Ties together query expansion, BM25/vector search, RRF fusion,
reranking, and score blending into a single async pipeline that
React can subscribe to for progressive UI updates.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

src/pipeline/orchestrator.ts

@@ -0,0 +1,118 @@
+import type {
+  Chunk,
+  EmbeddedChunk,
+  ScoredChunk,
+  ExpandedQuery,
+  PipelineEvent,
+} from "../types";
+import { expandQuery } from "./expansion";
+import { embedQuery } from "./embeddings";
+import { vectorSearch } from "./vectorSearch";
+import { BM25Index } from "./bm25";
+import { reciprocalRankFusion } from "./rrf";
+import { scoreDocument } from "./rerank";
+import { blendScores } from "./blend";
+
+export interface PipelineInput {
+  query: string;
+  chunks: Chunk[];
+  embeddedChunks: EmbeddedChunk[];
+  bm25Index: BM25Index;
+}
+
+export async function* runPipeline(
+  input: PipelineInput,
+): AsyncGenerator<PipelineEvent> {
+  const { query, chunks, embeddedChunks, bm25Index } = input;
+
+  // ── Stage 1: Query Expansion ──
+  yield { stage: "expansion", status: "running" };
+  let expanded: ExpandedQuery;
+  try {
+    expanded = await expandQuery(query);
+    yield { stage: "expansion", status: "done", data: expanded };
+  } catch (err) {
+    yield { stage: "expansion", status: "error", error: String(err) };
+    return;
+  }
+
+  // ── Stage 2: Parallel Search ──
+  yield { stage: "search", status: "running" };
+
+  // BM25 searches:
+  //   1. Original query -> BM25 (dashed line in architecture diagram)
+  //   2. Lex keywords -> BM25
+  //   3. Vec sentences -> BM25
+  const bm25Original = bm25Index.search(query, 20);
+  const bm25Lex = bm25Index.search(expanded.lex, 20);
+  const bm25Vec = expanded.vec.flatMap((v) => bm25Index.search(v, 20));
+
+  // Vector searches:
+  //   1. HyDE -> Vector Search
+  //   2. Vec sentences -> Vector Search
+  const hydeEmbedding = await embedQuery(expanded.hyde);
+  const vecEmbeddings = await Promise.all(
+    expanded.vec.map((v) => embedQuery(v)),
+  );
+
+  const vecHyde = vectorSearch(hydeEmbedding, embeddedChunks, 20);
+  const vecVec = vecEmbeddings.flatMap((emb) =>
+    vectorSearch(emb, embeddedChunks, 20),
+  );
+
+  // Combine all hits for UI display
+  const allBm25: ScoredChunk[] = [...bm25Original, ...bm25Lex, ...bm25Vec];
+  const allVector: ScoredChunk[] = [...vecHyde, ...vecVec];
+
+  yield {
+    stage: "search",
+    status: "done",
+    data: { bm25Hits: allBm25, vectorHits: allVector },
+  };
+
+  // ── Stage 3: RRF Fusion ──
+  // Build ranked lists for RRF (order matters for weights: first 2 get 2x)
+  const rrfLists = [
+    { results: bm25Original, queryType: "original" as const, query },
+    { results: vecHyde, queryType: "hyde" as const, query: expanded.hyde },
+    { results: bm25Lex, queryType: "lex" as const, query: expanded.lex },
+    ...expanded.vec.map((v, i) => ({
+      results: vecVec.slice(i * 20, (i + 1) * 20),
+      queryType: "vec" as const,
+      query: v,
+    })),
+    ...expanded.vec.map((v, i) => ({
+      results: bm25Vec.slice(i * 20, (i + 1) * 20),
+      queryType: "vec" as const,
+      query: v,
+    })),
+  ];
+
+  const rrfResults = reciprocalRankFusion(rrfLists);
+  yield { stage: "rrf", status: "done", data: { merged: rrfResults } };
+
+  // ── Stage 4: Reranking ──
+  yield { stage: "rerank", status: "running" };
+  const rerankScores = new Map<string, number>();
+  for (const result of rrfResults) {
+    const score = await scoreDocument(query, result.bestChunk);
+    rerankScores.set(result.docId, score);
+  }
+
+  // Build reranked view for "before/after" visualization
+  const rerankedResults = rrfResults.map((r) => ({
+    ...r,
+    rerankScore: rerankScores.get(r.docId) ?? 0,
+    blendedScore: 0, // computed in blend step
+  }));
+
+  yield {
+    stage: "rerank",
+    status: "done",
+    data: { before: rrfResults, after: rerankedResults },
+  };
+
+  // ── Stage 5: Score Blending ──
+  const finalResults = blendScores(rrfResults, rerankScores);
+  yield { stage: "blend", status: "done", data: { finalResults } };
+}