Blame for src/data/conceptsData.ts

export type ConceptCategory = 'all' | 'lexical' | 'neural' | 'hybrid' | 'eval';

export interface ConceptExampleStep {
  title: string;
  mathExpression?: string;
  explanation: string;
  result?: string;
}

export interface ConceptExample {
  title: string;
  sourceLesson: string; // e.g. "Lesson 5: Conceptual Framework" or "Lesson 6: NFCorpus Baseline"
  lessonLink?: string;
  query: string;
  documentId?: string;
  documentSnippet?: string;
  plainContext: string;
  steps: ConceptExampleStep[];
  codeSnippet?: {
    language: string;
    code: string;
    description: string;
  };
  takeaway: string;
}

export interface ConceptIllustration {
  imageSrc?: string;
  badge: string;
  caption: string;
  sourceDoc: string;
}

export interface ConceptItem {
  id: string;
  title: string;
  shortTitle: string;
  category: 'lexical' | 'neural' | 'hybrid' | 'eval';
  acronym?: string;
  oneLiner: string;
  plainSummary: string;
  whyItMatters: string;
  formula: {
    latex: string;
    params: { name: string; desc: string }[];
  };
  visualizationType:
    | 'posting-chain'
    | 'bm25-saturation'
    | 'bi-encoder-mips'
    | 'latent-mips'
    | 'hnsw-layers'
    | 'splade-expansion'
    | 'cross-vs-bi'
    | 'rrf-fusion'
    | 'ndcg-discount'
    | 'mrr-reciprocal'
    | 'qrels-breakdown';
  illustration?: ConceptIllustration;
  example: ConceptExample;
  castoriniContext: string;
  tags: string[];
}

export const CONCEPTS_DATA: ConceptItem[] = [
  {
    id: 'bm25',
    title: 'BM25 (Best Matching 25) & RSJ-IDF',
    shortTitle: 'BM25 Lexical',
    category: 'lexical',
    acronym: 'BM25',
    oneLiner: 'The classic keyword scoring formula balancing term importance, repetition saturation, and document length.',
    plainSummary:
      'Search engines must decide two things: (1) How important is a word in the collection? and (2) Does this document mention it often enough? BM25 awards huge points to rare words (like "deen") and low points to common words (like "what"). It also caps the score so repeating a word 20 times doesn\'t give 20× the points, and penalizes long, rambling documents.',
    whyItMatters:
      'BM25 is the battle-tested first-stage retriever in almost all search engines. It is fast, requires no neural network training, and excels at exact keyword matching.',
    formula: {
      latex: '\\text{Score}(q, d) = \\sum_{t \\in q} \\ln\\left(1 + \\frac{N - DF_t + 0.5}{DF_t + 0.5}\\right) \\cdot \\frac{TF(t, d) \\cdot (k_1 + 1)}{TF(t, d) + k_1 \\cdot \\left(1 - b + b \\cdot \\frac{|d|}{\\text{avgdl}}\\right)}',
      params: [
        { name: 'N', desc: 'Total documents in corpus (e.g. 8,841,823 in MS MARCO)' },
        { name: 'DF_t', desc: 'Document frequency: how many documents contain term t' },
        { name: 'TF(t, d)', desc: 'Term frequency: count of term t in this specific document' },
        { name: 'k₁ (≈ 0.9)', desc: 'Controls how fast term frequency score levels off' },
        { name: 'b (≈ 0.4)', desc: 'Length penalty: penalizes documents longer than average' },
        { name: '|d| / avgdl', desc: 'Relative passage length compared to corpus average' },
      ],
    },
    visualizationType: 'bm25-saturation',
    illustration: {
      imageSrc: '/diagrams/architecture-bm25.png',
      badge: 'Lesson 5 Onboarding Diagram',
      caption: 'From Pyserini Conceptual Framework: BM25 viewed as a Bi-Encoder where the query is a multi-hot vector and the document is a sparse lexical vector.',
      sourceDoc: 'pyserini/docs/conceptual-framework.md',
    },
    example: {
      title: 'Manual BM25 Score Verification from Lesson 5',
      sourceLesson: 'Lesson 5: Conceptual Framework (MS MARCO Passage)',
      lessonLink: 'https://github.com/castorini/pyserini/blob/master/docs/conceptual-framework.md',
      query: 'what is paula deen\'s brother',
      documentId: '7187158',
      documentSnippet: 'Paula Deen and her brother Earl W. Bubba Hiers are being sued by a former general manager at Uncle Bubba\'s Seafood and Oyster House...',
      plainContext:
        'In Lesson 5, you learn how Lucene computes BM25 scores by extracting term weights directly from the index for Doc 7187158. Here is the exact calculation in plain numbers.',
      steps: [
        {
          title: 'Query Stemming & Tokenization',
          explanation: 'Lucene removes stopwords ("is") and stems words. The query becomes 4 tokens: "what", "paula", "deen", "brother".',
          result: 'Tokens: [what, paula, deen, brother]',
        },
        {
          title: 'Word Rarity (RSJ-IDF) & Frequency (TF)',
          explanation:
            '"deen" appears in only 2,314 docs out of 8.8M (very rare) → weight 7.42. "paula" appears in 4,108 docs → weight 6.44. "brother" appears in 19,482 docs → weight 4.09. "what" appears in 1.4M docs (very common) and 0 times in this doc → weight 0.0.',
          result: 'deen=+7.42, paula=+6.44, brother=+4.09, what=0.0',
        },
        {
          title: 'Summing the Term Weights',
          mathExpression: '\\text{Score} = 7.41972 + 6.43852 + 4.09124 = 17.94948',
          explanation: 'Summing the non-zero matching term weights gives the exact score produced by Lucene.',
          result: 'Total BM25 Score = 17.94950 (Rank #1 in Lucene)',
        },
      ],
      codeSnippet: {
        language: 'python',
        code: `from pyserini.index.lucene import LuceneIndexReader

index_reader = LuceneIndexReader('indexes/lucene-index-msmarco-passage')
tf = index_reader.get_document_vector('7187158')
bm25_weights = {term: index_reader.compute_bm25_term_weight('7187158', term) for term in tf.keys()}

# Score query tokens against document weights:
query_tokens = ['what', 'paula', 'deen', 'brother']
score = sum(bm25_weights.get(t, 0) for t in query_tokens)
print(f"BM25 Score: {score:.5f}") # 17.94950`,
        description: 'Exact Python code from Lesson 5 reproducing the query-document score.',
      },
      takeaway: 'Doc 7187158 ranks #1 because it matches the two rarest words ("deen" and "paula"). Repeating common words cannot overcome high-IDF matches.',
    },
    castoriniContext:
      'In Pyserini, default Lucene parameters are tuned for passage retrieval (k₁=0.9, b=0.4), achieving MRR@10 of ~0.187 on MS MARCO dev.small.',
    tags: ['BM25', 'Robertson-Spärck Jones', 'TF-IDF', 'Lexical', 'Lucene', 'Pyserini'],
  },
  {
    id: 'inverted-index',
    title: 'Inverted Index & Postings',
    shortTitle: 'Inverted Index',
    category: 'lexical',
    acronym: 'Postings',
    oneLiner: 'The book-index data structure mapping each word to a sorted list of documents containing it.',
    plainSummary:
      'Think of an index at the back of a textbook: instead of reading every page from start to finish to find "mitochondria", you look up the word and immediately jump to pages 42, 89, and 115. An inverted index does the same for millions of documents, storing sorted document IDs, term counts, and word positions.',
    whyItMatters:
      'Without an inverted index, searching 8.8 million documents would require checking every text linearly (taking seconds or minutes per query). With postings lists, Lucene intersects only relevant documents in 1 to 3 milliseconds.',
    formula: {
      latex: '\\text{Posting}(t) = \\left[ (d_1, tf_1, [pos]), (d_2, tf_2, [pos]), \\dots, (d_k, tf_k, [pos]) \\right]',
      params: [
        { name: 't', desc: 'Vocabulary term in the Lucene dictionary' },
        { name: 'd_i', desc: 'Sorted document ID stored with delta compression (d_i - d_{i-1})' },
        { name: 'tf_i', desc: 'Term frequency of t in document d_i' },
        { name: 'pos', desc: 'Token positions enabling phrase searches like "paula deen"' },
      ],
    },
    visualizationType: 'posting-chain',
    example: {
      title: 'Intersecting Postings Lists for Query "paula deen"',
      sourceLesson: 'Lesson 1 & 4: Inverted Indexing in Lucene / Anserini',
      lessonLink: 'https://github.com/castorini/anserini/blob/master/docs/start-here.md',
      query: 'what is paula deen\'s brother',
      documentId: '7187158',
      documentSnippet: 'Paula Deen and her brother Earl W. Bubba Hiers...',
      plainContext:
        'When you run an Anserini index on MS MARCO, Lucene constructs posting lists for every word in the dictionary.',
      steps: [
        {
          title: 'Dictionary Lookup',
          explanation: 'Lucene looks up the word dictionary (FST) for "deen" and "brother", finding their exact byte positions on disk.',
          result: 'Instant O(1) dictionary pointer lookup.',
        },
        {
          title: 'Postings List Traversal',
          explanation: 'Lucene traverses the postings list for "deen" ([120, 4092, ... 7187158]) and "brother" ([310, ... 7187158]).',
          result: 'Uses skip pointers to leap over thousands of non-matching docIDs.',
        },
        {
          title: 'Block-Max WAND Pruning',
          explanation: 'Each block of 128 postings stores maximum possible impact scores. If a block cannot beat the current top-10 heap cutoff, it is skipped entirely without unpacking.',
          result: 'Over 95% of candidates pruned without decompression.',
        },
      ],
      codeSnippet: {
        language: 'python',
        code: `from pyserini.index.lucene import LuceneIndexReader

reader = LuceneIndexReader('indexes/lucene-index-msmarco-passage')
postings = reader.get_postings_list('deen')
print(f"Total documents containing 'deen': {len(postings)}")
print(f"First hit docid: {postings[0].docid}, tf={postings[0].tf}")`,
        description: 'Extracting posting lists from a Lucene index using Pyserini.',
      },
      takeaway: 'Inverted posting lists transform brute-force O(|Corpus|) scans into efficient O(|Postings|) list intersections with skip pointers.',
    },
    castoriniContext:
      'Anserini builds highly optimized Apache Lucene inverted indexes with Block-Max WAND (BMW) traversal and impact score sorting for state-of-the-art BM25 execution.',
    tags: ['Lucene', 'Postings', 'Lexical', 'Anserini', 'Block-Max WAND', 'Compression'],
  },
  {
    id: 'bi-encoder-framework',
    title: 'The Unified Bi-Encoder & MIPS Framework',
    shortTitle: 'Unified Bi-Encoder',
    category: 'hybrid',
    acronym: 'MIPS Framework',
    oneLiner: 'The conceptual framework showing how lexical, dense, and learned sparse models all share the exact same inner-product objective.',
    plainSummary:
      'In his 2021 framework paper, Prof. Jimmy Lin showed that all major first-stage search models follow the same blueprint: (1) encode the query into a vector, (2) encode the document into a vector, and (3) compute relevance as their dot product. BM25, Dense neural models, and SPLADE are not fundamentally different concepts—they are just different types of vectors inside this dual-encoder blueprint.',
    whyItMatters:
      'Demystifies information retrieval: instead of seeing BM25, embeddings, and vector databases as separate worlds, you understand them as points along a shared mathematical spectrum.',
    formula: {
      latex: '\\text{Score}(q, d) = \\langle E_Q(q), E_D(d) \\rangle = \\sum_{i} E_Q(q)_i \\cdot E_D(d)_i',
      params: [
        { name: 'E_Q(q)', desc: 'Query representation vector (sparse multi-hot, dense embedding, or learned logits)' },
        { name: 'E_D(d)', desc: 'Document representation vector (BM25 weights, dense embedding, or SPLADE weights)' },
        { name: '⟨ · , · ⟩', desc: 'Standard inner product (dot product) yielding the scalar relevance score' },
      ],
    },
    visualizationType: 'bi-encoder-mips',
    illustration: {
      imageSrc: '/diagrams/architecture-biencoder.png',
      badge: 'Lesson 5 Foundation Diagram',
      caption: 'The universal bi-encoder architecture: Query Encoder + Document Encoder + Comparison Function (Inner Product).',
      sourceDoc: 'pyserini/docs/conceptual-framework.md',
    },
    example: {
      title: 'Comparing the 3 Paradigms on the Same Query',
      sourceLesson: 'Lesson 5: A Conceptual Framework for Retrieval',
      lessonLink: 'https://github.com/castorini/pyserini/blob/master/docs/conceptual-framework.md',
      query: 'what is paula deen\'s brother',
      documentId: '7187158',
      documentSnippet: 'Paula Deen and her brother Earl W. Bubba Hiers...',
      plainContext:
        'Lesson 5 explicitly walks through how BM25, Dense, and Learned Sparse fit into the exact same dot-product equation.',
      steps: [
        {
          title: 'Sparse Lexical (BM25)',
          explanation: 'Dimensions: |V| ≈ 500,000 words. Sparsity: > 99.9% zeros. Unsupervised. Index: Inverted posting list.',
          result: '⟨E_Q, E_D⟩ evaluated via posting list intersection in ~1.5ms.',
        },
        {
          title: 'Dense Learned (BGE / Contriever)',
          explanation: 'Dimensions: d = 768 continuous numbers. Sparsity: 0% zeros (dense). Supervised transformer. Index: HNSW proximity graph.',
          result: '⟨E_Q, E_D⟩ evaluated via vector beam search in ~3ms.',
        },
        {
          title: 'Learned Sparse (SPLADE)',
          explanation: 'Dimensions: |V| = 30,522 WordPiece tokens. Sparsity: ~95% zeros. Neural MLM expansion. Index: Lucene Impact index.',
          result: '⟨E_Q, E_D⟩ evaluated via Lucene quantized impact postings.',
        },
      ],
      codeSnippet: {
        language: 'python',
        code: `# Jimmy Lin's Unified Bi-Encoder Formulation:
# In BM25:
dot_bm25 = sum(multihot_q[t] * bm25_d[t] for t in q_tokens & d_tokens)

# In Dense BGE:
dot_dense = np.dot(bge_query_vec, bge_doc_vec)

# Both optimize argmax_d ⟨E_Q(q), E_D(d)⟩!`,
        description: 'Both paradigms compute the exact same inner product operator over different vector representations.',
      },
      takeaway: 'All first-stage search algorithms are instantiations of the bi-encoder architecture optimizing Maximum Inner Product Search (MIPS).',
    },
    castoriniContext:
      'Formulated in Jimmy Lin\'s landmark paper "A Proposed Conceptual Framework for a Representational Approach to Information Retrieval" (arXiv:2110.01529).',
    tags: ['Bi-Encoder', 'MIPS', 'Representation', 'Foundations', 'Theory', 'Jimmy Lin'],
  },
  {
    id: 'dense-bi-encoder',
    title: 'Dense Bi-Encoder & MIPS',
    shortTitle: 'Dense Bi-Encoder',
    category: 'neural',
    acronym: 'BGE / MIPS',
    oneLiner: 'Dual neural encoders mapping text into 768-dimensional semantic vectors where distance equals meaning.',
    plainSummary:
      'Instead of counting matching keywords, a neural transformer maps the query into a 768-number vector, and another transformer maps documents into 768-number vectors. If two texts mean similar things—even using completely different words—their vectors point in the same direction in geometric space.',
    whyItMatters:
      'Solves the classical "vocabulary mismatch" problem: a query for "heart attack causes" will match a medical paper discussing "myocardial infarction etiology" even if they share zero identical words.',
    formula: {
      latex: '\\text{Score}(q, d) = \\langle E_Q(q), E_D(d) \\rangle = \\sum_{i=1}^{768} E_Q(q)_i \\cdot E_D(d)_i = \\cos(\\theta) \\quad (\\text{when normalized})',
      params: [
        { name: 'E_Q(q)', desc: '768-D normalized embedding generated by query transformer' },
        { name: 'E_D(d)', desc: '768-D precomputed document embedding stored in vector index' },
        { name: 'MIPS', desc: 'Maximum Inner Product Search: find the vector with highest dot product' },
      ],
    },
    visualizationType: 'latent-mips',
    illustration: {
      imageSrc: '/diagrams/architecture-dense.png',
      badge: 'Lesson 5 & 7 Onboarding Diagram',
      caption: 'Dense retrieval architecture: query and documents encoded by transformers into dense semantic embeddings.',
      sourceDoc: 'pyserini/docs/conceptual-framework2.md',
    },
    example: {
      title: 'Manual Dense Dot Product on NFCorpus (PLAIN-3074 & MED-4555)',
      sourceLesson: 'Lesson 6 & 7: Deeper Dive into Dense Representations',
      lessonLink: 'https://github.com/castorini/pyserini/blob/master/docs/conceptual-framework2.md',
      query: 'How to Help Prevent Abdominal Aortic Aneurysms',
      documentId: 'MED-4555',
      documentSnippet: 'Analysis of risk factors for abdominal aortic aneurysm in a cohort of more than 3 million individuals... Ultrasound screening can reduce mortality... smoking cessation and a healthy lifestyle...',
      plainContext:
        'In Lesson 7, you reconstruct the 768-D dense vector from Faiss for doc MED-4555 and encode query PLAIN-3074 to verify the dot product by hand.',
      steps: [
        {
          title: 'Document Embedding Lookup in Faiss',
          explanation: 'Pyserini reads the NFCorpus Faiss index and extracts the precomputed 768-number embedding for MED-4555.',
          result: 'v1 = [+0.0381, -0.0194, +0.0712, ..., -0.0429] (768 continuous floats)',
        },
        {
          title: 'Query Encoding with BGE-base',
          explanation: 'The query is passed through BAAI/bge-base-en-v1.5 with mean pooling and L2 normalization.',
          result: 'v_q = [+0.0412, -0.0182, +0.0695, ..., -0.0410] (768 continuous floats)',
        },
        {
          title: 'Inner Product Calculation',
          mathExpression: '\\text{Score} = \\sum_{i=1}^{768} v_q[i] \\cdot v_1[i] = 0.791379',
          explanation: 'Taking the dot product yields 0.791379, exactly matching FaissSearcher hit #1.',
          result: 'Rank #1 Score = 0.791379 (Exact match to Lesson 7)',
        },
      ],
      codeSnippet: {
        language: 'python',
        code: `from pyserini.search.faiss import FaissSearcher
from pyserini.encode import AutoQueryEncoder

encoder = AutoQueryEncoder('BAAI/bge-base-en-v1.5', device='cpu', pooling='mean', l2_norm=True)
searcher = FaissSearcher('indexes/nfcorpus.bge-base-en-v1.5', encoder)
hits = searcher.search('How to Help Prevent Abdominal Aortic Aneurysms')

print(f"Top Hit: {hits[0].docid} score={hits[0].score:.6f}") # MED-4555 0.791379`,
        description: 'Executing dense retrieval in Pyserini on NFCorpus from Lesson 6.',
      },
      takeaway: 'Dense models map semantic concepts into geometry: MED-4555 is top-ranked because its vector points almost directly in the same direction as the query.',
    },
    castoriniContext:
      'Pyserini integrates Faiss and Lucene 9+ HNSW with BAAI/bge-base-en-v1.5 and Contriever for state-of-the-art dense baselines.',
    tags: ['Dense', 'MIPS', 'BGE', 'Bi-Encoder', 'Faiss', 'Embeddings', 'NFCorpus'],
  },
  {
    id: 'hnsw',
    title: 'HNSW (Hierarchical Navigable Small World)',
    shortTitle: 'HNSW Vector Graph',
    category: 'neural',
    acronym: 'HNSW',
    oneLiner: 'A multi-layer proximity graph algorithm enabling sub-5ms nearest-neighbor search over dense vectors.',
    plainSummary:
      'Because dense vector coordinates are continuous numbers without exact zeros, traditional inverted posting lists cannot be used. HNSW organizes vectors like an express highway system: top layers have long-distance hops across clusters, while bottom layers connect close neighbors for fine-grained search.',
    whyItMatters:
      'Allows searching across tens of millions of 768-D vectors in 2 to 5 milliseconds instead of doing slow brute-force scans over the entire dataset.',
    formula: {
      latex: '\\text{Search Complexity: } \\mathcal{O}(\\log N) \\quad \\text{vs. Brute-Force Scan: } \\mathcal{O}(N \\cdot D)',
      params: [
        { name: 'N', desc: 'Total indexed vectors (e.g. 8.8M in MS MARCO)' },
        { name: 'D', desc: 'Vector dimensionality (e.g. 768 in BGE)' },
        { name: 'M (e.g. 16)', desc: 'Max bi-directional connection links per node' },
        { name: 'efSearch (e.g. 64)', desc: 'Dynamic candidate queue size during traversal' },
      ],
    },
    visualizationType: 'hnsw-layers',
    example: {
      title: 'HNSW Graph Traversal on NFCorpus Dense Index',
      sourceLesson: 'Lesson 3 & 7: Dense Retrieval with Faiss / Lucene HNSW',
      lessonLink: 'https://github.com/castorini/anserini/blob/master/docs/experiments-msmarco-passage2.md',
      query: 'How to Help Prevent Abdominal Aortic Aneurysms',
      documentId: 'MED-4555',
      documentSnippet: 'Analysis of risk factors for abdominal aortic aneurysm...',
      plainContext:
        'When searching the NFCorpus dense index, HNSW finds the nearest vector (MED-4555) in only 14 vector distance calculations instead of checking all 3,633 documents.',
      steps: [
        {
          title: 'Layer 2 (Express Highway)',
          explanation: 'Starts at an entry node. Greedily hops to the node closest to query vector in the sparse global layer.',
          result: 'Hops: Node #102 → Node #240. Descends to Layer 1.',
        },
        {
          title: 'Layer 1 (Sub-regional Clustering)',
          explanation: 'Follows local connections closer into the semantic neighborhood of cardiovascular health.',
          result: 'Hops: Node #240 → Node #1809 → Node #3210. Descends to Layer 0.',
        },
        {
          title: 'Layer 0 (Ground Mesh, efSearch=64)',
          explanation: 'Executes beam search maintaining a priority queue of 64 closest candidates, locking onto MED-4555.',
          result: 'Finds MED-4555 (Score: 0.791379) in just 14 vector distance checks!',
        },
      ],
      codeSnippet: {
        language: 'python',
        code: `import faiss

# Reading an HNSW Faiss index:
index = faiss.read_index('indexes/nfcorpus.bge-base-en-v1.5/index')
print(f"Total Vectors: {index.ntotal}") # 3633 vectors in NFCorpus
hnsw_index = faiss.downcast_index(index)
print(f"HNSW efSearch: {hnsw_index.hnsw.efSearch}")`,
        description: 'Inspecting Faiss HNSW index parameters in Pyserini.',
      },
      takeaway: 'HNSW replaces linear scans of O(N·D) with logarithmic graph hops, making large-scale vector search practical in production.',
    },
    castoriniContext:
      'Castorini benchmarks both Faiss HNSW and native Lucene 9+ FlatVectorsFormat HNSW extensively to optimize latency-recall tradeoffs.',
    tags: ['HNSW', 'ANN', 'Graph', 'Faiss', 'Lucene', 'Vector Search'],
  },
  {
    id: 'splade',
    title: 'SPLADE (Sparse Lexical and Expansion Model)',
    shortTitle: 'SPLADE Learned Sparse',
    category: 'neural',
    acronym: 'SPLADE-v3',
    oneLiner: 'Neural language models predicting non-negative expansion weights over a 30,522-token WordPiece vocabulary.',
    plainSummary:
      'SPLADE uses a transformer (BERT MLM head) to read a passage and predict weights for all 30,522 words in its vocabulary. It automatically adds relevant synonyms (expansion) while leaving irrelevant words at zero. These learned weights are then quantized and stored in a standard Lucene inverted index!',
    whyItMatters:
      'Combines the synonym intelligence of neural networks with the lightning speed and lightweight memory footprint of Lucene inverted indexes.',
    formula: {
      latex: 'w_j(d) = \\max_{t \\in d} \\log\\left(1 + \\text{ReLU}\\left(W_j \\cdot h_t + b_j\\right)\\right) + \\lambda_{\\text{sparse}} \\cdot \\mathcal{L}_{\\text{reg}}',
      params: [
        { name: 'w_j(d)', desc: 'Learned scalar impact weight for vocabulary term j in document d' },
        { name: 'h_t', desc: 'Contextual hidden state for token t from transformer encoder' },
        { name: 'W_j, b_j', desc: 'Weights of the BERT Masked Language Model (MLM) head' },
        { name: 'Quantization', desc: 'Weights are multiplied (e.g. × 100) and stored as integer TFs in Lucene' },
      ],
    },
    visualizationType: 'splade-expansion',
    example: {
      title: 'Neural Expansion on NFCorpus with SPLADE-v3',
      sourceLesson: 'Lesson 8: Learned Sparse Representations with SPLADE-v3',
      lessonLink: 'https://github.com/castorini/pyserini/blob/master/docs/conceptual-framework3.md',
      query: 'How to Help Prevent Abdominal Aortic Aneurysms',
      documentId: 'MED-4555',
      documentSnippet: 'Analysis of risk factors for abdominal aortic aneurysm in a cohort of more than 3 million individuals...',
      plainContext:
        'In Lesson 8, you encode NFCorpus with SPLADE-v3 and build a Lucene Impact Index using `--impact --pretokenized`.',
      steps: [
        {
          title: 'Input Text Evaluation',
          explanation: 'Passage text is fed into the SPLADE-v3 transformer encoder.',
          result: 'Evaluates contextual interactions across all tokens.',
        },
        {
          title: 'Vocabulary Logits & Expansion',
          explanation: 'The model activates related medical terms: "aneurysm" (8.9), "aortic" (8.1), "+vascular" (3.4), "+ultrasound" (2.8).',
          result: 'Expands document with synonyms not originally written in the text.',
        },
        {
          title: 'Quantization into Lucene TFs',
          mathExpression: '\\text{Synthetic TF} = \\text{round}(w_j \\cdot 100)',
          explanation: 'Floating-point weights are multiplied by 100 and stored as integer term frequencies in Lucene.',
          result: 'Indexed with `pyserini.index.lucene --impact --pretokenized`.',
        },
      ],
      codeSnippet: {
        language: 'bash',
        code: `# SPLADE indexing command in Pyserini from Lesson 8:
python -m pyserini.index.lucene \\
  --collection JsonVectorCollection \\
  --input encode/nfcorpus.splade-v3 \\
  --index indexes/nfcorpus.splade-v3 \\
  --generator DefaultLuceneDocumentGenerator \\
  --impact \\
  --pretokenized`,
        description: 'Building a Lucene Impact index for SPLADE-v3 encoded passages.',
      },
      takeaway: 'SPLADE turns neural embeddings into synthetic inverted posting lists, achieving dense-like semantic matching at inverted index speeds.',
    },
    castoriniContext:
      'Castorini pioneered "Lucene Impact Search" (Lin & Trotman, 2020), enabling Anserini to index and retrieve SPLADE, uniCOIL, and TILDE natively.',
    tags: ['SPLADE', 'Learned Sparse', 'Impact Search', 'Anserini', 'Expansion', 'uniCOIL'],
  },
  {
    id: 'cross-encoder',
    title: 'Cross-Encoder & RankLLM Rerankers',
    shortTitle: 'Cross-Encoder (RankLLM)',
    category: 'neural',
    acronym: 'monoT5 / RankLLM',
    oneLiner: 'Joint-attention transformer architectures where query and passage tokens attend directly to each other in a single context.',
    plainSummary:
      'Unlike bi-encoders which process query and document in separate silos, a cross-encoder feeds query and passage tokens together into a single transformer. Every word in the query directly compares itself to every word in the passage, allowing the model to spot negations, subtle details, and exact context.',
    whyItMatters:
      'Yields the highest ranking precision in information retrieval, serving as the definitive second-stage reranker over top-50 or top-100 first-stage candidates.',
    formula: {
      latex: 'P(\\text{rel} \\mid q, d) = \\text{Softmax}\\left(\\text{Transformer}([\\text{CLS}], q, [\\text{SEP}], d, [\\text{SEP}])\\right)_{\\text{true}}',
      params: [
        { name: '[CLS], [SEP]', desc: 'Special delimiter tokens separating query and passage in a single context' },
        { name: 'Full Attention', desc: 'Every query token attends directly to every document token simultaneously' },
        { name: 'Two-Stage Cascade', desc: 'Stage 1 (BM25/BGE): 8.8M → Top 100. Stage 2 (RankLLM): Top 100 → Top 10.' },
      ],
    },
    visualizationType: 'cross-vs-bi',
    example: {
      title: 'Two-Stage Retrieval Cascade with RankLLM',
      sourceLesson: 'Onboarding Notes: RankLLM & monoT5 Reranking Path',
      lessonLink: 'https://github.com/castorini/rank_llm',
      query: 'what is paula deen\'s brother',
      documentId: '7187158',
      documentSnippet: 'Paula Deen and her brother Earl W. Bubba Hiers are being sued by a former general manager...',
      plainContext:
        'In the onboarding notes, Prof. Lin highlights RankLLM as the research group\'s open-source reranking toolkit.',
      steps: [
        {
          title: 'Stage 1: Fast Candidate Retrieval',
          explanation: 'BM25 or BGE retrieves top 100 candidate passages from 8.8 million in ~10ms.',
          result: 'Top 100 candidate pool with high recall (>85%).',
        },
        {
          title: 'Stage 2: Joint Token Attention with RankLLM',
          explanation: 'RankLLM concatenates query and passage. Every word in the query attends to "brother", "Earl", "Bubba", and "sued".',
          result: 'Deep semantic interaction matrix across all tokens.',
        },
        {
          title: 'Listwise Ranking Permutation',
          explanation: 'RankLLM reorders the 100 candidates based on explicit relevance log-probabilities.',
          result: 'Passage 7187158 jumps to Rank #1 with highest confidence.',
        },
      ],
      codeSnippet: {
        language: 'python',
        code: `from rank_llm.rerank import Reranker

# RankLLM listwise reranker over first-stage candidates:
reranker = Reranker.create_reranker(model_path="castorini/rank_vicuna_7b_v1")
reranked_results = reranker.rerank(
    query="what is paula deen's brother",
    candidates=first_stage_hits[:100]
)
print(f"Top 1: {reranked_results[0].docid}")`,
        description: 'Reranking first-stage candidates with Castorini\'s RankLLM toolkit.',
      },
      takeaway: 'Cross-encoders trade computational throughput for unmatched ranking precision, forming the second stage of modern search engines.',
    },
    castoriniContext:
      'Castorini pioneered monoBERT, monoT5 (Nogueira & Lin, 2019), and RankLLM (Ronak Pradeep et al.), powering open-source neural rerankers.',
    tags: ['Cross-Encoder', 'RankLLM', 'monoT5', 'Attention', 'Reranking', 'Two-Stage'],
  },
  {
    id: 'rrf',
    title: 'Reciprocal Rank Fusion (RRF)',
    shortTitle: 'RRF Hybrid Fusion',
    category: 'hybrid',
    acronym: 'RRF',
    oneLiner: 'An unsupervised, robust rank-aggregation method fusing outputs from heterogeneous retrieval paradigms.',
    plainSummary:
      'BM25 produces scores like 17.9, while dense retrievers produce cosine scores like 0.79. Because their score scales are incompatible, combining them directly requires messy calibration. RRF ignores the raw scores entirely and uses only each document\'s rank position, summing 1/(60 + rank).',
    whyItMatters:
      'Guarantees superior hybrid search performance (BM25 + Dense + SPLADE) with zero machine learning training, zero score tuning, and immunity to score outliers.',
    formula: {
      latex: '\\text{RRF\\_Score}(d) = \\sum_{m \\in \\mathcal{M}} \\frac{1}{k + r_m(d)} \\quad \\text{where } k = 60',
      params: [
        { name: 'm ∈ M', desc: 'Set of independent retriever runs (e.g. BM25 and BGE Dense)' },
        { name: 'r_m(d)', desc: 'Rank position (1, 2, 3...) of document d in retriever m' },
        { name: 'k (default: 60)', desc: 'Smoothing constant that dampens score differences between top positions' },
      ],
    },
    visualizationType: 'rrf-fusion',
    example: {
      title: 'Fusing BM25 & BGE Dense Runs on MS MARCO Query',
      sourceLesson: 'Lesson 5 & 8: Hybrid Search & Fusion in Pyserini',
      lessonLink: 'https://github.com/castorini/pyserini/blob/master/docs/conceptual-framework.md',
      query: 'what is paula deen\'s brother',
      documentId: '7187158',
      documentSnippet: 'Paula Deen and her brother Earl W. Bubba Hiers...',
      plainContext:
        'In Pyserini, reciprocal rank fusion is built in to combine BM25 runs with Faiss dense runs.',
      steps: [
        {
          title: 'BM25 Retriever Run',
          mathExpression: 'S_{\\text{BM25}} = \\frac{1}{60 + 1} = \\frac{1}{61} \\approx 0.016393',
          explanation: 'Doc 7187158 is retrieved at Rank #1 by BM25 (exact keyword match).',
          result: 'BM25 Reciprocal Rank: 0.016393',
        },
        {
          title: 'BGE Dense Retriever Run',
          mathExpression: 'S_{\\text{Dense}} = \\frac{1}{60 + 2} = \\frac{1}{62} \\approx 0.016129',
          explanation: 'Doc 7187158 is retrieved at Rank #2 by BGE Dense (semantic match).',
          result: 'Dense Reciprocal Rank: 0.016129',
        },
        {
          title: 'Fused RRF Aggregation',
          mathExpression: '\\text{RRF}(7187158) = 0.016393 + 0.016129 = 0.032522',
          explanation: 'Documents appearing near the top of both lists receive dominant combined scores, securing rank #1 overall.',
          result: 'Final Fused Rank #1 (score = 0.032522)',
        },
      ],
      codeSnippet: {
        language: 'python',
        code: `from pyserini.fusion import reciprocal_rank_fusion

# Fusing BM25 run and Dense run with k=60:
fused_run = reciprocal_rank_fusion(
    runs=[bm25_run, dense_run],
    k=60
)
print("Top Fused Hit:", fused_run['1048585'][0])`,
        description: 'Using Pyserini built-in fusion module to combine heterogeneous runs.',
      },
      takeaway: 'RRF provides zero-parameter hybrid search that almost always outperforms either BM25 or Dense search in isolation.',
    },
    castoriniContext:
      'Pyserini provides native hybrid fusion modules combining Lucene BM25 runs with HNSW dense runs via RRF with zero friction.',
    tags: ['Hybrid', 'RRF', 'Fusion', 'Rank Aggregation', 'Pyserini'],
  },
  {
    id: 'mrr',
    title: 'MRR (Mean Reciprocal Rank)',
    shortTitle: 'MRR@10 Metric',
    category: 'eval',
    acronym: 'MRR@10',
    oneLiner: 'Evaluation metric measuring how quickly a user finds the first correct answer.',
    plainSummary:
      'If the first relevant document is at Rank #1, the score is 1.0 (1/1). If it is at Rank #2, it is 0.5 (1/2); at Rank #3, it is 0.33 (1/3). If no relevant document appears within the top 10, the score is 0. The scores are averaged across all test queries.',
    whyItMatters:
      'The definitive benchmark metric for single-answer query corpora like MS MARCO Passage Ranking.',
    formula: {
      latex: '\\text{MRR@K} = \\frac{1}{|Q|} \\sum_{i=1}^{|Q|} \\frac{1}{\\text{rank}_i} \\quad (\\text{where } \\text{rank}_i \\le K, \\text{ else } 0)',
      params: [
        { name: '|Q|', desc: 'Total queries in evaluation set (6,980 for MS MARCO dev.small)' },
        { name: 'rank_i', desc: 'Rank position (1 to K) of the first relevant document found for query i' },
        { name: 'K (typically 10)', desc: 'Cutoff threshold: documents retrieved below position K contribute 0' },
      ],
    },
    visualizationType: 'mrr-reciprocal',
    example: {
      title: 'Evaluating MS MARCO Dev Queries with MRR@10',
      sourceLesson: 'Lesson 2 & 4: Evaluating MS MARCO Passage Ranking',
      lessonLink: 'https://github.com/castorini/anserini/blob/master/docs/experiments-msmarco-passage.md',
      query: 'Batch of 3 Queries from MS MARCO dev.small',
      documentId: 'Q1, Q2, Q3',
      documentSnippet: 'Official trec_eval evaluation on MS MARCO dev runs...',
      plainContext:
        'In Lesson 2 and 4, you run trec_eval to compute MRR@10 across the 6,980 dev queries.',
      steps: [
        {
          title: 'Query 1 (Paula Deen Brother)',
          mathExpression: 'RR_1 = \\frac{1}{1} = 1.000',
          explanation: 'Relevant doc 7187158 retrieved at Rank #1.',
          result: 'RR = 1.000',
        },
        {
          title: 'Query 2 (Androgen Receptor Define)',
          mathExpression: 'RR_2 = \\frac{1}{3} \\approx 0.333',
          explanation: 'Relevant doc 4339068 retrieved at Rank #3.',
          result: 'RR = 0.333',
        },
        {
          title: 'Query 3 (Hard Informational Query)',
          mathExpression: 'RR_3 = 0.000',
          explanation: 'Relevant doc retrieved at Rank #12 (below cutoff of K=10).',
          result: 'RR = 0.000 (missed top-10)',
        },
        {
          title: 'Mean Reciprocal Rank Across 3 Queries',
          mathExpression: '\\text{MRR@10} = \\frac{1.000 + 0.333 + 0.000}{3} = 0.444',
          explanation: 'Averaging the reciprocal ranks yields the overall benchmark score.',
          result: 'MRR@10 = 0.444',
        },
      ],
      codeSnippet: {
        language: 'bash',
        code: `# Official MS MARCO trec_eval command from Lesson 2 & 4:
python -m pyserini.eval.trec_eval \\
  -c -M 10 -m recip_rank \\
  collections/msmarco-passage/qrels.dev.small.tsv \\
  runs/run.msmarco-passage.bm25.txt`,
        description: 'Running official MRR@10 evaluation in Pyserini using trec_eval.',
      },
      takeaway: 'MRR heavily rewards models that place the ground truth answer directly into rank position #1.',
    },
    castoriniContext:
      'Official MS MARCO leaderboards use `trec_eval -c -M 10 -m recip_rank`, where Castorini baselines (BM25: 0.187, BGE: 0.358, SPLADE: 0.380, RankLLM: 0.415) set key references.',
    tags: ['Evaluation', 'MRR', 'MS MARCO', 'trec_eval', 'Metrics', 'recip_rank'],
  },
  {
    id: 'ndcg',
    title: 'nDCG (Normalized Discounted Cumulative Gain)',
    shortTitle: 'nDCG@10 Metric',
    category: 'eval',
    acronym: 'nDCG@10',
    oneLiner: 'Evaluation metric for search results with multiple graded relevance levels (0, 1, or 2).',
    plainSummary:
      'In medical or scientific search, documents are not simply "relevant or not"—they have degrees of relevance (e.g. 0=Irrelevant, 1=Somewhat relevant, 2=Highly relevant). nDCG sums the points of each returned document with a logarithmic discount for lower ranks, divided by the score of a perfect ordering (IDCG).',
    whyItMatters:
      'The gold-standard evaluation metric for medical, scientific, and enterprise search (e.g. NFCorpus, TREC-COVID, BEIR benchmark) where multiple relevant documents exist.',
    formula: {
      latex: '\\text{DCG@K} = \\sum_{i=1}^K \\frac{2^{\\text{rel}_i} - 1}{\\log_2(i + 1)}, \\qquad \\text{nDCG@K} = \\frac{\\text{DCG@K}}{\\text{IDCG@K}}',
      params: [
        { name: 'rel_i', desc: 'Graded relevance rating (0, 1, or 2) of document at position i' },
        { name: 'log₂(i + 1)', desc: 'Discount factor penalizing relevant results pushed further down the list' },
        { name: 'IDCG@K', desc: 'Ideal DCG: maximum possible score achievable if results were sorted perfectly' },
      ],
    },
    visualizationType: 'ndcg-discount',
    illustration: {
      imageSrc: '/diagrams/nfcorpus-atlas-vis.png',
      badge: 'Lesson 6 Onboarding Diagram',
      caption: 'NFCorpus Nomic Atlas 2D semantic embedding map from Lesson 6, demonstrating medical document clusters evaluated with nDCG@10.',
      sourceDoc: 'pyserini/docs/experiments-nfcorpus.md',
    },
    example: {
      title: 'Evaluating NFCorpus Query PLAIN-2 with Graded Qrels',
      sourceLesson: 'Lesson 6 & 7: Dense & Sparse Evaluation on NFCorpus',
      lessonLink: 'https://github.com/castorini/pyserini/blob/master/docs/experiments-nfcorpus.md',
      query: 'Do Cholesterol Statin Drugs Cause Breast Cancer?',
      documentId: 'PLAIN-2',
      documentSnippet: 'Ground truth: MED-10 (rel=2), MED-14 (rel=2), MED-826 (rel=1), MED-2429 (rel=0)...',
      plainContext:
        'In Lesson 6, you evaluate BGE-base retrieval on NFCorpus using nDCG@10.',
      steps: [
        {
          title: 'System Returned Ranking at Top 3',
          mathExpression: '\\text{DCG@3} = \\frac{2^1 - 1}{\\log_2(2)} + \\frac{2^2 - 1}{\\log_2(3)} + \\frac{2^0 - 1}{\\log_2(4)} = 1.0 + 1.893 + 0 = 2.893',
          explanation: 'Retriever returns: Rank 1 (rel=1), Rank 2 (rel=2), Rank 3 (rel=0).',
          result: 'System DCG@3 = 2.893',
        },
        {
          title: 'Ideal Ranking (IDCG@3)',
          mathExpression: '\\text{IDCG@3} = \\frac{2^2 - 1}{\\log_2(2)} + \\frac{2^1 - 1}{\\log_2(3)} + 0 = 3.0 + 0.631 = 3.631',
          explanation: 'Perfect ranking sorts by highest grade first: Rank 1 (rel=2), Rank 2 (rel=1), Rank 3 (rel=0).',
          result: 'Ideal IDCG@3 = 3.631',
        },
        {
          title: 'Normalized Discounted Cumulative Gain',
          mathExpression: '\\text{nDCG@3} = \\frac{2.893}{3.631} = 0.7967',
          explanation: 'Dividing actual DCG by ideal IDCG produces a normalized score between 0.0 and 1.0.',
          result: 'nDCG@3 = 0.7967 (79.7% of optimal ranking)',
        },
      ],
      codeSnippet: {
        language: 'bash',
        code: `# Official NFCorpus nDCG@10 trec_eval command from Lesson 6:
python -m pyserini.eval.trec_eval \\
  -c -m ndcg_cut.10 \\
  collections/nfcorpus/qrels.test.tsv \\
  runs/run.nfcorpus.bge.txt`,
        description: 'Running official nDCG@10 evaluation in Pyserini on NFCorpus.',
      },
      takeaway: 'nDCG accounts for both graded relevance levels and rank position, providing a comprehensive assessment of multi-document retrieval quality.',
    },
    castoriniContext:
      'Standard benchmark metric across BEIR benchmark evaluations and the NFCorpus dataset included in this studio.',
    tags: ['Evaluation', 'nDCG', 'BEIR', 'NFCorpus', 'trec_eval', 'Metrics', 'Graded Relevance'],
  },
  {
    id: 'qrels',
    title: 'Qrels & Ground Truth Relevance Judgments',
    shortTitle: 'Qrels & Relevance',
    category: 'eval',
    acronym: 'qrels',
    oneLiner: 'The standard TREC judgment files recording curated human annotations of document relevance for queries.',
    plainSummary:
      'In scientific information retrieval, a system\'s output cannot be evaluated without objective truth. A "qrels" file contains historical human judgment lines specifying: QueryID, iteration (typically "0" or "Q0"), DocID, and relevance grade. Evaluation programs like trec_eval compare system run files against qrels to compute metrics.',
    whyItMatters:
      'Prevents subjective confirmation bias, enables reproducible science, and powers the entire benchmark infrastructure of modern NLP and search.',
    formula: {
      latex: '\\text{Record Format: } \\langle\\text{query\\_id}\\rangle \\quad 0 \\quad \\langle\\text{doc\\_id}\\rangle \\quad \\langle\\text{relevance\\_grade}\\rangle',
      params: [
        { name: 'query_id', desc: 'Identifies the target search query in the benchmark collection' },
        { name: 'iteration (0 or Q0)', desc: 'Historical TREC round identifier (usually fixed to 0)' },
        { name: 'doc_id', desc: 'Identifier of the evaluated passage or document in the collection' },
        { name: 'relevance_grade', desc: 'Binary (0 or 1) for MS MARCO; Graded (0, 1, or 2) for NFCorpus' },
      ],
    },
    visualizationType: 'qrels-breakdown',
    illustration: {
      imageSrc: '/diagrams/Prometheus-Model.png',
      badge: 'Start Here Onboarding Diagram',
      caption: 'From Anserini Start Here: How retrieval evaluation and test collections power modern answer generation and RAG pipelines.',
      sourceDoc: 'anserini/docs/start-here.md',
    },
    example: {
      title: 'Structure of MS MARCO and NFCorpus Qrels Files',
      sourceLesson: 'Anserini: Start Here & Lesson 2: Tour of MS MARCO',
      lessonLink: 'https://github.com/castorini/anserini/blob/master/docs/start-here.md',
      query: 'Query 1048585 (what is paula deen\'s brother)',
      documentId: '7187158',
      documentSnippet: 'Line in qrels.dev.small.tsv: 1048585 0 7187158 1',
      plainContext:
        'In the Start Here guide, you unpack `qrels.dev.small.tsv` to understand how automated ranking evaluation works.',
      steps: [
        {
          title: 'Line 1: MS MARCO Passage Dev Qrels',
          mathExpression: '\\texttt{1048585} \\quad 0 \\quad \\texttt{7187158} \\quad 1',
          explanation: 'Specifies that for query 1048585, doc 7187158 is judged relevant (1).',
          result: 'Ground truth target established.',
        },
        {
          title: 'Line 2: NFCorpus Graded Qrels',
          mathExpression: '\\texttt{PLAIN-2} \\quad \\text{Q0} \\quad \\texttt{MED-10} \\quad 2',
          explanation: 'In NFCorpus, relevance is graded: 2 denotes highly relevant, 1 denotes partially relevant, 0 denotes irrelevant.',
          result: 'Graded multi-level relevance ground truth.',
        },
        {
          title: 'Comparison with Run File',
          explanation: 'A retrieval system outputs a run file: <query_id> Q0 <doc_id> <rank> <score> <tag>. trec_eval matches run against qrels.',
          result: 'trec_eval scores exact alignment between system predictions and human ground truth.',
        },
      ],
      codeSnippet: {
        language: 'python',
        code: `# Inspecting qrels in Python:
with open('collections/msmarco-passage/qrels.dev.small.tsv') as f:
    for line in f:
        qid, iter_tag, docid, rel = line.strip().split()
        if int(rel) > 0:
            print(f"Query {qid} -> Relevant Doc {docid} (Grade {rel})")
            break`,
        description: 'Parsing standard TREC qrels records.',
      },
      takeaway: 'Qrels files are the empirical bedrock of information retrieval research; without them, no metric can be objectively computed.',
    },
    castoriniContext:
      'Castorini actively maintains pre-packaged qrels and run evaluation scripts through Pyserini (`pyserini.eval.trec_eval`).',
    tags: ['Evaluation', 'qrels', 'trec_eval', 'Ground Truth', 'Benchmarking', 'TREC'],
  },
];