eval/qa_15_starter.jsonl

{"id": "q001", "question": "What is RAG?", "gold_answer": "RAG stands for Retrieval-Augmented Generation. It is a method where the system first retrieves relevant passages from a document corpus and then uses those passages as context to generate an answer.", "relevant_chunk_ids": ["77d45130-baac-4209-b554-544014951db9_page_9_chunk_1", "77d45130-baac-4209-b554-544014951db9_page_9_chunk_2"], "expected_terms": ["retrieval", "augmented", "generation"], "difficulty": "easy"}
{"id": "q002", "question": "What does BM25 do in a RAG system?", "gold_answer": "BM25 is a keyword-based retrieval algorithm that ranks documents by term frequency and inverse document frequency. It is used in RAG systems for fast, interpretable text search without requiring vector embeddings.", "relevant_chunk_ids": ["77d45130-baac-4209-b554-544014951db9_page_12_chunk_4", "77d45130-baac-4209-b554-544014951db9_page_13_chunk_1"], "expected_terms": ["BM25", "keyword", "retrieval", "term"], "difficulty": "easy"}
{"id": "q003", "question": "What is the purpose of chunking in document processing?", "gold_answer": "Chunking splits documents into smaller passages so that the retrieval system can find and return the most relevant pieces of text rather than entire documents.", "relevant_chunk_ids": ["77d45130-baac-4209-b554-544014951db9_page_27_chunk_1", "77d45130-baac-4209-b554-544014951db9_page_27_chunk_2"], "expected_terms": ["chunk", "split", "text", "retrieval"], "difficulty": "easy"}
{"id": "q004", "question": "What is a knowledge graph?", "gold_answer": "A knowledge graph represents entities and their relationships as nodes and edges, enabling multi-hop reasoning over structured facts extracted from documents.", "relevant_chunk_ids": ["77d45130-baac-4209-b554-544014951db9_page_42_chunk_1", "77d45130-baac-4209-b554-544014951db9_page_42_chunk_2"], "expected_terms": ["knowledge", "graph", "entities", "relationships"], "difficulty": "easy"}
{"id": "q005", "question": "What is entity extraction?", "gold_answer": "Entity extraction is the process of identifying important named entities such as people, organizations, dates, and concepts from text, making documents queryable by name.", "relevant_chunk_ids": ["77d45130-baac-4209-b554-544014951db9_page_40_chunk_1", "77d45130-baac-4209-b554-544014951db9_page_40_chunk_2"], "expected_terms": ["entity", "extraction", "named", "text"], "difficulty": "easy"}
{"id": "q006", "question": "How does hybrid retrieval combine keyword and semantic search?", "gold_answer": "Hybrid retrieval combines BM25 keyword matching with vector-based semantic search, using weighted scores from both methods to rank results. This captures both exact term matches and semantic meaning.", "relevant_chunk_ids": ["77d45130-baac-4209-b554-544014951db9_page_12_chunk_4", "77d45130-baac-4209-b554-544014951db9_page_13_chunk_1"], "expected_terms": ["hybrid", "keyword", "semantic", "BM25", "vector"], "difficulty": "medium"}
{"id": "q007", "question": "What chunking strategies exist and what are their trade-offs?", "gold_answer": "Strategies include fixed-size chunking, sentence-based splitting, sliding window with overlap, and parent-child hybrid. Fixed-size is simple but cuts mid-thought. Sentence-based preserves grammar but varies in size. Sliding window preserves cross-boundary context but creates more chunks.", "relevant_chunk_ids": ["77d45130-baac-4209-b554-544014951db9_page_27_chunk_2", "77d45130-baac-4209-b554-544014951db9_page_27_chunk_3", "77d45130-baac-4209-b554-544014951db9_page_27_chunk_4", "77d45130-baac-4209-b554-544014951db9_page_28_chunk_1"], "expected_terms": ["fixed", "sentence", "sliding", "overlap", "chunk"], "difficulty": "medium"}
{"id": "q008", "question": "What metadata fields are stored with each chunk and why?", "gold_answer": "Each chunk stores chunk_id as primary key, document_id for linking, section_title for context, page_number for source location, start_char and end_char for window expansion, and has_table flag for retrieval ranking.", "relevant_chunk_ids": ["77d45130-baac-4209-b554-544014951db9_page_28_chunk_3", "77d45130-baac-4209-b554-544014951db9_page_29_chunk_1"], "expected_terms": ["chunk_id", "page_number", "section", "metadata"], "difficulty": "medium"}
{"id": "q009", "question": "How does entity normalisation work and why is it needed?", "gold_answer": "Entity normalisation maps different surface forms of the same entity to a single canonical form, for example mapping TCS to Tata Consultancy Services. This prevents duplicate entries and ensures consistent entity retrieval.", "relevant_chunk_ids": ["77d45130-baac-4209-b554-544014951db9_page_41_chunk_2", "77d45130-baac-4209-b554-544014951db9_page_41_chunk_3"], "expected_terms": ["normalisation", "entity", "canonical", "mapping"], "difficulty": "medium"}
{"id": "q010", "question": "What is the database design for a vectorless RAG system?", "gold_answer": "The database uses a relational schema with tables for documents, chunks, entities, and their relationships. SQLite is used for its simplicity. The schema connects documents to chunks, chunks to entities, and entities to each other through typed relationships.", "relevant_chunk_ids": ["77d45130-baac-4209-b554-544014951db9_page_30_chunk_1", "77d45130-baac-4209-b554-544014951db9_page_30_chunk_2", "77d45130-baac-4209-b554-544014951db9_page_30_chunk_3"], "expected_terms": ["database", "schema", "SQLite", "relational", "chunks"], "difficulty": "medium"}
{"id": "q011", "question": "How does the knowledge graph enable multi-hop reasoning?", "gold_answer": "The knowledge graph connects entities through typed relationships like belongs_to, replaced_by, and related_to. Multi-hop queries traverse these edges to find connections that span multiple documents or sections, which keyword search alone cannot discover.", "relevant_chunk_ids": ["77d45130-baac-4209-b554-544014951db9_page_42_chunk_1", "77d45130-baac-4209-b554-544014951db9_page_42_chunk_2", "77d45130-baac-4209-b554-544014951db9_page_43_chunk_1"], "expected_terms": ["multi-hop", "graph", "entities", "relationships", "reasoning"], "difficulty": "hard"}
{"id": "q012", "question": "What are the trade-offs between different entity extraction methods?", "gold_answer": "Regex patterns are deterministic and fast but require manual effort and miss novel entities. spaCy NER handles unknown entities but needs a trained model. The hybrid approach combines spaCy NER with custom EntityRuler patterns for both statistical and deterministic extraction.", "relevant_chunk_ids": ["77d45130-baac-4209-b554-544014951db9_page_40_chunk_3", "77d45130-baac-4209-b554-544014951db9_page_41_chunk_1", "77d45130-baac-4209-b554-544014951db9_page_41_chunk_2"], "expected_terms": ["regex", "spaCy", "NER", "extraction", "entity"], "difficulty": "hard"}
{"id": "q013", "question": "How does the retrieval router decide which retrieval strategy to use?", "gold_answer": "The retrieval router classifies the query type and routes it to the appropriate retrieval method. Entity queries go to the entity retriever, metadata queries use metadata filtering, exact phrase queries use BM25 with phrase matching, and general questions use hybrid search.", "relevant_chunk_ids": ["77d45130-baac-4209-b554-544014951db9_page_39_chunk_1", "77d45130-baac-4209-b554-544014951db9_page_39_chunk_2"], "expected_terms": ["retrieval", "router", "query", "entity", "BM25"], "difficulty": "hard"}
{"id": "q014", "question": "Compare NetworkX and Neo4j for knowledge graph storage. When would you choose each?", "gold_answer": "NetworkX stores the graph in-memory as a Python object, making it simple and suitable for graphs up to roughly 10 million nodes. Neo4j is a separate disk-based server that handles billions of nodes but requires infrastructure management. NetworkX is chosen for prototyping and CPU-only environments.", "relevant_chunk_ids": ["77d45130-baac-4209-b554-544014951db9_page_43_chunk_2", "77d45130-baac-4209-b554-544014951db9_page_43_chunk_3"], "expected_terms": ["NetworkX", "Neo4j", "graph", "memory", "storage"], "difficulty": "hard"}
{"id": "q015", "question": "What problems does BM25 length normalisation cause for RAG retrieval and how can they be addressed?", "gold_answer": "BM25 length normalisation heavily favours short documents, so very short chunks with exact matches may be retrieved even if they contain almost no useful context. Very long chunks are penalised even when highly relevant. This can be addressed by using consistent chunk sizes and tuning the BM25 b parameter.", "relevant_chunk_ids": ["77d45130-baac-4209-b554-544014951db9_page_28_chunk_2", "77d45130-baac-4209-b554-544014951db9_page_12_chunk_4"], "expected_terms": ["BM25", "length", "normalisation", "chunk", "retrieval"], "difficulty": "hard"}