eval/qa_50_graphresearcher.jsonl

{"id": "q001", "question": "What is RAG?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_1"], "expected_terms": ["retrieval", "augmented", "generation"], "difficulty": "easy"}
{"id": "q002", "question": "What does RAG stand for?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_2"], "expected_terms": ["retrieval-augmented", "generation"], "difficulty": "easy"}
{"id": "q003", "question": "What is a vector database?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_3"], "expected_terms": ["vector", "database", "embeddings"], "difficulty": "easy"}
{"id": "q004", "question": "What is chunking in document processing?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_4"], "expected_terms": ["chunk", "text", "split"], "difficulty": "easy"}
{"id": "q005", "question": "What is an embedding model?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_5"], "expected_terms": ["embedding", "model", "vector"], "difficulty": "easy"}
{"id": "q006", "question": "What is a knowledge graph?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_6"], "expected_terms": ["knowledge", "graph", "entities", "relations"], "difficulty": "easy"}
{"id": "q007", "question": "What is semantic search?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_7"], "expected_terms": ["semantic", "search", "meaning"], "difficulty": "easy"}
{"id": "q008", "question": "What is BM25?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_8"], "expected_terms": ["bm25", "keyword", "retrieval"], "difficulty": "easy"}
{"id": "q009", "question": "What is a cross-encoder?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_9"], "expected_terms": ["cross-encoder", "reranking"], "difficulty": "easy"}
{"id": "q010", "question": "What is document indexing?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_10"], "expected_terms": ["index", "document", "search"], "difficulty": "easy"}
{"id": "q011", "question": "What is hallucination in LLMs?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_11"], "expected_terms": ["hallucination", "incorrect", "generated"], "difficulty": "easy"}
{"id": "q012", "question": "What is a citation in a RAG system?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_12"], "expected_terms": ["citation", "source", "reference"], "difficulty": "easy"}
{"id": "q013", "question": "What is context in answer generation?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_13"], "expected_terms": ["context", "retrieved", "generation"], "difficulty": "easy"}
{"id": "q014", "question": "What is hybrid retrieval?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_14"], "expected_terms": ["hybrid", "keyword", "vector"], "difficulty": "easy"}
{"id": "q015", "question": "What is a document graph?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_15"], "expected_terms": ["document", "graph", "entities"], "difficulty": "easy"}
{"id": "q016", "question": "How does hybrid retrieval combine keyword and vector search?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_16"], "expected_terms": ["hybrid", "keyword", "vector", "retrieval"], "difficulty": "medium"}
{"id": "q017", "question": "Explain how reranking improves retrieval quality.", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_17"], "expected_terms": ["reranking", "relevance", "cross-encoder"], "difficulty": "medium"}
{"id": "q018", "question": "How are entities extracted from document chunks?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_18"], "expected_terms": ["entity", "extraction", "chunk"], "difficulty": "medium"}
{"id": "q019", "question": "What is the role of the knowledge graph in answering questions?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_19"], "expected_terms": ["graph", "context", "entities", "relations"], "difficulty": "medium"}
{"id": "q020", "question": "How does the system ground its answers in source documents?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_20"], "expected_terms": ["citation", "source", "grounded"], "difficulty": "medium"}
{"id": "q021", "question": "How does the entity normalization process work?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_21"], "expected_terms": ["normalization", "entity", "lowercase"], "difficulty": "medium"}
{"id": "q022", "question": "How are relations detected between entities?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_22"], "expected_terms": ["relation", "co-occurrence", "sentence"], "difficulty": "medium"}
{"id": "q023", "question": "What is graph-guided retrieval?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_23"], "expected_terms": ["graph", "guided", "retrieval", "chunks"], "difficulty": "medium"}
{"id": "q024", "question": "How does the system handle noisy entities?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_24"], "expected_terms": ["noisy", "filter", "stop", "entity"], "difficulty": "medium"}
{"id": "q025", "question": "How does the evidence extraction step work?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_25"], "expected_terms": ["evidence", "extraction", "sentence"], "difficulty": "medium"}
{"id": "q026", "question": "What happens when the LLM answer is not good enough?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_26"], "expected_terms": ["fallback", "quality", "check"], "difficulty": "medium"}
{"id": "q027", "question": "How does the prompt builder create grounded prompts?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_27"], "expected_terms": ["prompt", "grounded", "evidence"], "difficulty": "medium"}
{"id": "q028", "question": "What is the role of the source viewer?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_28"], "expected_terms": ["source", "viewer", "verification"], "difficulty": "medium"}
{"id": "q029", "question": "How does document comparison work?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_29"], "expected_terms": ["comparison", "document", "side-by-side"], "difficulty": "medium"}
{"id": "q030", "question": "What information does the admin dashboard show?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_30"], "expected_terms": ["admin", "dashboard", "monitoring"], "difficulty": "medium"}
{"id": "q031", "question": "Compare the retrieval results with and without graph-guided retrieval.", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_31"], "expected_terms": ["graph", "retrieval", "fusion", "chunks"], "difficulty": "hard"}
{"id": "q032", "question": "What are the limitations of rule-based entity extraction?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_32"], "expected_terms": ["rule-based", "entity", "limitation"], "difficulty": "hard"}
{"id": "q033", "question": "How does the graph fusion strategy decide which chunks to boost or add?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_33"], "expected_terms": ["fusion", "graph", "score", "boost"], "difficulty": "hard"}
{"id": "q034", "question": "If two documents discuss the same topic, how does comparison mode surface differences?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_34"], "expected_terms": ["comparison", "document", "differences"], "difficulty": "hard"}
{"id": "q035", "question": "How does the system filter noisy entities from the graph?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_35"], "expected_terms": ["noisy", "filter", "quality", "entity"], "difficulty": "hard"}
{"id": "q036", "question": "What is the effect of graph_entity_limit on answer quality?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_36"], "expected_terms": ["graph_entity_limit", "quality", "entities"], "difficulty": "hard"}
{"id": "q037", "question": "How does the graph context text get injected into the prompt?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_37"], "expected_terms": ["graph", "context", "prompt", "evidence"], "difficulty": "hard"}
{"id": "q038", "question": "What determines the weight of a relation in the graph?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_38"], "expected_terms": ["weight", "relation", "co-occurrence"], "difficulty": "hard"}
{"id": "q039", "question": "How are graph-supported chunks scored differently from normal chunks?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_39"], "expected_terms": ["graph", "score", "boost", "supported"], "difficulty": "hard"}
{"id": "q040", "question": "What trade-offs exist between graph retrieval top-k and answer latency?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_40"], "expected_terms": ["top_k", "latency", "trade-off"], "difficulty": "hard"}
{"id": "q041", "question": "How does the verb relation map affect edge labeling?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_41"], "expected_terms": ["verb", "relation", "edge", "label"], "difficulty": "hard"}
{"id": "q042", "question": "How does the system prevent relation explosion in dense chunks?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_42"], "expected_terms": ["relation", "explosion", "limit", "combinations"], "difficulty": "hard"}
{"id": "q043", "question": "What is the impact of chunk size on entity extraction quality?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_43"], "expected_terms": ["chunk", "size", "entity", "extraction"], "difficulty": "hard"}
{"id": "q044", "question": "How would switching from rule-based to NER-based extraction change the graph?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_44"], "expected_terms": ["ner", "rule-based", "extraction", "graph"], "difficulty": "hard"}
{"id": "q045", "question": "Under what conditions does graph-guided retrieval not improve results?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_45"], "expected_terms": ["graph", "retrieval", "no improvement"], "difficulty": "hard"}
{"id": "q046", "question": "How does the answer quality checker decide to use the fallback?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_46"], "expected_terms": ["quality", "checker", "fallback", "threshold"], "difficulty": "hard"}
{"id": "q047", "question": "What is the role of mention_count in entity ranking?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_47"], "expected_terms": ["mention_count", "entity", "ranking"], "difficulty": "hard"}
{"id": "q048", "question": "How are chunk_ids linked between graph entities and retrieval results?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_48"], "expected_terms": ["chunk_id", "graph", "retrieval", "link"], "difficulty": "hard"}
{"id": "q049", "question": "What happens when graph context returns zero matched entities?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_49"], "expected_terms": ["zero", "entities", "fallback", "graph"], "difficulty": "hard"}
{"id": "q050", "question": "How could the evaluation framework be extended with human judges?", "gold_answer": "TODO: fill with verified answer from your document", "relevant_chunk_ids": ["TODO_chunk_id_50"], "expected_terms": ["evaluation", "human", "judges", "annotation"], "difficulty": "hard"}