Spaces:

amer003100
/

vectorless_RAG

Running

vectorless_RAG / evaluation /eval_dataset.json
AI Assistant
docs: add quantitative evaluation benchmarks (80-question single-doc test) and report
b3242a4 5 days ago
61.9 kB
[
  {
    "id": "LQ_01",
    "category": "Local",
    "question": "What are the four distinct categories of Knowledge Graphs categorized by knowledge type in Section 2?",
    "ground_truth_nodes": [
      "0002"
    ],
    "ground_truth_pages": [
      3,
      4
    ],
    "ground_truth_answer": "In Section 2, Knowledge Graphs are categorized into four types based on knowledge: 1) Encyclopedia Knowledge Graphs (e.g., Wikidata, DBpedia, YAGO, Freebase), 2) Commonsense Knowledge Graphs (e.g., ConceptNet, ATOMIC), 3) Domain-specific Knowledge Graphs (e.g., UMLS, SNOMED CT), and 4) Multimodal Knowledge Graphs (e.g., IMGpedia, MMKG).",
    "key_facts": [
      "Encyclopedia Knowledge Graphs",
      "Commonsense Knowledge Graphs",
      "Domain-specific Knowledge Graphs",
      "Multimodal Knowledge Graphs"
    ]
  },
  {
    "id": "LQ_02",
    "category": "Local",
    "question": "How does ERNIE (Zhang et al.) integrate entity embeddings into pre-trained language models in Section 4.1?",
    "ground_truth_nodes": [
      "0005"
    ],
    "ground_truth_pages": [
      7
    ],
    "ground_truth_answer": "ERNIE fuses entity representations into text tokens by utilizing knowledge-graph embeddings (TransE) and passing tokens and entities through dedicated multi-layer fusion encoders (mutually attentive stacked transformers) to align linguistic and knowledge spaces.",
    "key_facts": [
      "ERNIE",
      "TransE or entity embeddings",
      "multi-layer fusion encoder / mutually attentive",
      "aligns textual and entity representations"
    ]
  },
  {
    "id": "LQ_03",
    "category": "Local",
    "question": "What is the difference between PaE (Pre-trained as Encoders) and PaD (Pre-trained as Decoders) in Knowledge Graph Completion (Section 5.2)?",
    "ground_truth_nodes": [
      "0011"
    ],
    "ground_truth_pages": [
      12,
      13
    ],
    "ground_truth_answer": "In PaE (Pre-trained as Encoders), LLMs serve as representation encoders that produce entity/relation embeddings, which are scored using prediction heads or scoring functions. In PaD (Pre-trained as Decoders), LLMs generate target missing entity names directly in an auto-regressive generative sequence-to-sequence manner via prompting.",
    "key_facts": [
      "PaE uses LLM as encoder with scoring head",
      "PaD uses LLM as auto-regressive decoder",
      "PaD generates missing entities as text directly"
    ]
  },
  {
    "id": "LQ_04",
    "category": "Local",
    "question": "What are the three main steps of KG-enhanced LLM inference methods described in Section 4.2?",
    "ground_truth_nodes": [
      "0006"
    ],
    "ground_truth_pages": [
      9
    ],
    "ground_truth_answer": "The three main steps of KG-enhanced LLM inference are: 1) Knowledge Retrieval (identifying relevant entities/triples via entity linking or subgraph search), 2) Knowledge Representation (converting structured triples into text, graphs, or linear prompts), and 3) Knowledge Fusion (incorporating knowledge into the LLM during generation or attention).",
    "key_facts": [
      "Knowledge Retrieval",
      "Knowledge Representation",
      "Knowledge Fusion"
    ]
  },
  {
    "id": "LQ_05",
    "category": "Local",
    "question": "According to Appendix A, what are the primary advantages (pros) and disadvantages (cons) of Knowledge Graphs?",
    "ground_truth_nodes": [
      "0020"
    ],
    "ground_truth_pages": [
      27,
      28
    ],
    "ground_truth_answer": "KG pros include: Structural Knowledge (triples), High Accuracy (expert curated), Decisiveness, Interpretability (symbolic reasoning), Domain-specific Knowledge, and Evolving Knowledge. KG cons include: Incompleteness (hard to maintain), Lacking Language Understanding (ignores text nuances), and Difficulty handling Unseen Facts.",
    "key_facts": [
      "Structural Knowledge / Accuracy / Decisiveness / Interpretability",
      "Incompleteness",
      "Lacking Language Understanding",
      "Unseen Facts"
    ]
  },
  {
    "id": "LQ_06",
    "category": "Local",
    "question": "How does K-BERT incorporate knowledge graphs without retraining the base language model in Section 4.2?",
    "ground_truth_nodes": [
      "0006"
    ],
    "ground_truth_pages": [
      9
    ],
    "ground_truth_answer": "K-BERT injects triples directly into sentences to create knowledge-rich tree structures, and uses soft-position encoding alongside a visibility matrix (attention mask) during inference so that injected knowledge does not distort the original sentence meaning.",
    "key_facts": [
      "K-BERT",
      "visibility matrix",
      "soft-position encoding",
      "tree-structured sentence",
      "no retraining needed"
    ]
  },
  {
    "id": "LQ_07",
    "category": "Local",
    "question": "What are the two primary approaches for Knowledge Graph Embedding (KGE) using LLMs in Section 5.1?",
    "ground_truth_nodes": [
      "0010"
    ],
    "ground_truth_pages": [
      11,
      12
    ],
    "ground_truth_answer": "Section 5.1 classifies LLM-augmented KGE into: 1) LLM as Entity/Relation Feature Encoders (extracting contextual text representations of entity descriptions), and 2) Joint Structural and Textual Embedding (combining structural graph losses like TransE with language modeling objectives).",
    "key_facts": [
      "LLM as feature encoder",
      "Joint structural and textual embedding / combining text descriptions with KG structure"
    ]
  },
  {
    "id": "LQ_08",
    "category": "Local",
    "question": "What is the core concept of 'Synergized LLMs + KGs' as defined in Section 6?",
    "ground_truth_nodes": [
      "0015"
    ],
    "ground_truth_pages": [
      18
    ],
    "ground_truth_answer": "Synergized LLMs + KGs refers to a bidirectional, unified framework where LLMs and KGs play mutually beneficial and reciprocal roles in representation and reasoning, combining the perceptual and linguistic capabilities of LLMs with the structured factual rigor of KGs.",
    "key_facts": [
      "bidirectional unification / mutual enhancement",
      "synergized representation",
      "synergized reasoning",
      "combines perceptual LLM with structured KG"
    ]
  },
  {
    "id": "LQ_09",
    "category": "Local",
    "question": "In Section 4.4, how are Knowledge Graphs used to probe knowledge stored in Large Language Models?",
    "ground_truth_nodes": [
      "0008"
    ],
    "ground_truth_pages": [
      10
    ],
    "ground_truth_answer": "Section 4.4 describes KG-based probing (such as the LAMA benchmark) where factual triples (subject, relation, object) from KGs are converted into cloze-style prompt templates, and the LLM is evaluated on its accuracy in predicting the masked target entity without fine-tuning.",
    "key_facts": [
      "cloze-style prompt templates",
      "probing factual knowledge",
      "LAMA benchmark",
      "predicting masked object entity"
    ]
  },
  {
    "id": "LQ_10",
    "category": "Local",
    "question": "What specific sub-tasks are involved in Knowledge Graph Construction using LLMs in Section 5.3?",
    "ground_truth_nodes": [
      "0012"
    ],
    "ground_truth_pages": [
      14,
      15
    ],
    "ground_truth_answer": "The core sub-tasks of KG Construction are: 1) Entity Discovery (Named Entity Recognition, Entity Typing, and Entity Linking), 2) Relation Extraction (identifying relational predicates between entities), and 3) Event Extraction (identifying event triggers and event arguments).",
    "key_facts": [
      "Entity Discovery (NER, Entity Typing, Entity Linking)",
      "Relation Extraction",
      "Event Extraction"
    ]
  },
  {
    "id": "LQ_11",
    "category": "Local",
    "question": "What is the primary challenge in Knowledge-Graph-to-Text (KG-to-text) generation discussed in Section 5.4?",
    "ground_truth_nodes": [
      "0013"
    ],
    "ground_truth_pages": [
      16
    ],
    "ground_truth_answer": "The primary challenge is parallel data scarcity (scarcity of aligned KG triples and natural text descriptions) and ensuring verbalization fidelity, where LLMs must generate fluent descriptions without hallucinating unsupported claims beyond the graph.",
    "key_facts": [
      "parallel data scarcity",
      "scarcity of paired KG-text corpora",
      "generation fidelity / avoiding hallucinations"
    ]
  },
  {
    "id": "LQ_12",
    "category": "Local",
    "question": "What are the two major paradigms of Knowledge Graph Question Answering (KGQA) in Section 5.5?",
    "ground_truth_nodes": [
      "0014"
    ],
    "ground_truth_pages": [
      17
    ],
    "ground_truth_answer": "The two major paradigms in KGQA are: 1) Semantic Parsing (converting natural questions into executable formal queries like SPARQL or lambda-calculus), and 2) Information Retrieval (retrieving a relevant candidate subgraph or facts and directly extracting/generating the answer).",
    "key_facts": [
      "Semantic Parsing (SP-based)",
      "Information Retrieval (IR-based)",
      "SPARQL / formal queries",
      "subgraph extraction"
    ]
  },
  {
    "id": "LQ_13",
    "category": "Local",
    "question": "How does Section 2 define Commonsense Knowledge Graphs?",
    "ground_truth_nodes": [
      "0002"
    ],
    "ground_truth_pages": [
      4
    ],
    "ground_truth_answer": "Commonsense Knowledge Graphs model everyday knowledge, beliefs, causes, and behavioral rules about the physical and social world, representing events, intentions, and conceptual relations (e.g., ConceptNet, ATOMIC) beyond strict factual taxonomy.",
    "key_facts": [
      "everyday knowledge",
      "beliefs and causes",
      "physical and social world",
      "ConceptNet, ATOMIC"
    ]
  },
  {
    "id": "LQ_14",
    "category": "Local",
    "question": "How does the General Language Model (GLM) differ from BERT and GPT in the background discussion in Section 2?",
    "ground_truth_nodes": [
      "0002"
    ],
    "ground_truth_pages": [
      3
    ],
    "ground_truth_answer": "GLM employs auto-regressive blank filling that unifies auto-encoding (BERT-style) and auto-regressive (GPT-style) objectives, allowing variable-length text span corruption and generation for multi-task applications.",
    "key_facts": [
      "auto-regressive blank filling",
      "unifies auto-encoding and auto-regressive",
      "variable-length span corruption"
    ]
  },
  {
    "id": "LQ_15",
    "category": "Local",
    "question": "What are the five key disadvantages of Large Language Models identified in Section 1 and Appendix A?",
    "ground_truth_nodes": [
      "0001",
      "0020"
    ],
    "ground_truth_pages": [
      2,
      27
    ],
    "ground_truth_answer": "The five major limitations are: 1) Implicit Knowledge (uninterpretable parameter storage), 2) Hallucination (generating plausible but false statements), 3) Indecisiveness (stochastic probabilistic generation), 4) Black-box nature (lack of interpretability), and 5) Lack of Domain-specific/New Knowledge.",
    "key_facts": [
      "Implicit Knowledge",
      "Hallucination",
      "Indecisiveness",
      "Black-box",
      "Lacking Domain-specific/New Knowledge"
    ]
  },
  {
    "id": "LQ_16",
    "category": "Local",
    "question": "What are knowledge-guided training objectives in KG-enhanced pre-training (Section 4.1)?",
    "ground_truth_nodes": [
      "0005"
    ],
    "ground_truth_pages": [
      8
    ],
    "ground_truth_answer": "Knowledge-guided training objectives modify standard language modeling by designing entity masking (masking entire entity spans instead of random subwords), entity replacement detection, and entity-relation prediction tasks guided by KG structure.",
    "key_facts": [
      "entity masking",
      "entity replacement detection",
      "relation prediction",
      "guided by KG structure"
    ]
  },
  {
    "id": "LQ_17",
    "category": "Local",
    "question": "What are the two major sub-categories of Synergized LLMs + KGs discussed in Section 6?",
    "ground_truth_nodes": [
      "0015"
    ],
    "ground_truth_pages": [
      18,
      19
    ],
    "ground_truth_answer": "Section 6 categorizes Synergized LLMs + KGs into: 1) Synergized Knowledge Representation (joint pre-training and unified embeddings for text and graphs), and 2) Synergized Knowledge Reasoning (cooperative reasoning where LLMs act as cognitive agents and KGs act as symbolic verification engines).",
    "key_facts": [
      "Synergized Knowledge Representation",
      "Synergized Knowledge Reasoning"
    ]
  },
  {
    "id": "LQ_18",
    "category": "Local",
    "question": "According to Section 7.1, how can Knowledge Graphs be utilized to detect hallucinations in LLMs?",
    "ground_truth_nodes": [
      "0016"
    ],
    "ground_truth_pages": [
      20
    ],
    "ground_truth_answer": "KGs serve as an external grounded source of truth where generated statements from LLMs are converted into structured facts and verified against reliable KG entities and paths, checking for contradictions and unsupported entity relations.",
    "key_facts": [
      "external source of truth",
      "verifying generated statements against KG triples",
      "detecting contradictions and unsupported entity relations"
    ]
  },
  {
    "id": "LQ_19",
    "category": "Local",
    "question": "What are Multimodal Knowledge Graphs as described in Section 2?",
    "ground_truth_nodes": [
      "0002"
    ],
    "ground_truth_pages": [
      4
    ],
    "ground_truth_answer": "Multimodal Knowledge Graphs (MMKGs) extend conventional symbolic KGs by incorporating visual, audio, or sensory modalities, associating entities and relations with images, video, and audio assets (e.g., IMGpedia, MMKG).",
    "key_facts": [
      "incorporates images, audio, or visual modalities",
      "extends symbolic entities with multimedia",
      "IMGpedia, MMKG"
    ]
  },
  {
    "id": "LQ_20",
    "category": "Local",
    "question": "In Section 5.2, what is the inductive relation prediction problem in Knowledge Graph Completion?",
    "ground_truth_nodes": [
      "0011"
    ],
    "ground_truth_pages": [
      13
    ],
    "ground_truth_answer": "Inductive relation prediction involves predicting relations involving previously unseen entities that were not present during training, requiring language models to utilize textual descriptions and generalize beyond static transductive graph embeddings.",
    "key_facts": [
      "unseen entities not present during training",
      "predicting links for new nodes",
      "generalizing via textual descriptions"
    ]
  },
  {
    "id": "GQ_01",
    "category": "Global",
    "question": "What is the overarching categorization of research unifying LLMs and KGs proposed in this roadmap?",
    "ground_truth_nodes": [
      "0000",
      "0003",
      "0017"
    ],
    "ground_truth_pages": [
      1,
      5,
      21
    ],
    "ground_truth_answer": "The paper categorizes the entire research landscape into three unified frameworks: 1) KG-enhanced LLMs (injecting KGs into LLMs during pre-training, inference, or interpretability), 2) LLM-augmented KGs (using LLMs for KGE, KGC, KG construction, KG-to-text, and KGQA), and 3) Synergized LLMs + KGs (bidirectional unification in representation and reasoning).",
    "key_facts": [
      "three frameworks",
      "KG-enhanced LLMs",
      "LLM-augmented KGs",
      "Synergized LLMs + KGs"
    ]
  },
  {
    "id": "GQ_02",
    "category": "Global",
    "question": "What are the fundamental complementary strengths of LLMs versus KGs described throughout the paper?",
    "ground_truth_nodes": [
      "0000",
      "0001",
      "0020"
    ],
    "ground_truth_pages": [
      1,
      2,
      27,
      28
    ],
    "ground_truth_answer": "LLMs provide general knowledge, natural language understanding, generative fluency, and broad zero-shot transfer, but suffer from hallucination and black-box opacity. KGs provide structured, accurate, decisive, interpretable, and domain-specific knowledge, but lack natural language fluency and are often incomplete.",
    "key_facts": [
      "LLM: general knowledge, language fluency, generalization",
      "LLM limitations: hallucination, opacity",
      "KG: structural accuracy, interpretability, decisiveness",
      "KG limitations: incompleteness, lack of language understanding"
    ]
  },
  {
    "id": "GQ_03",
    "category": "Global",
    "question": "How does the paper categorize the integration of KGs into LLMs across the model lifecycle?",
    "ground_truth_nodes": [
      "0003",
      "0004",
      "0005",
      "0006",
      "0008"
    ],
    "ground_truth_pages": [
      5,
      7,
      9,
      10
    ],
    "ground_truth_answer": "The lifecycle integration covers: 1) Pre-training (enriching corpora and modifying training objectives with entity masks), 2) Inference (dynamic retrieval, prompting, and attention modifications without retraining), and 3) Interpretability (using KGs for knowledge probing and understanding model decisions).",
    "key_facts": [
      "Pre-training",
      "Inference",
      "Interpretability"
    ]
  },
  {
    "id": "GQ_04",
    "category": "Global",
    "question": "How do LLMs assist in addressing the incompleteness and sparseness of Knowledge Graphs across Section 5?",
    "ground_truth_nodes": [
      "0003",
      "0009",
      "0011",
      "0012"
    ],
    "ground_truth_pages": [
      5,
      11,
      12,
      14
    ],
    "ground_truth_answer": "LLMs leverage rich textual descriptions and generalized world knowledge to enrich KG embeddings, complete missing triples via generative sequence prediction (KGC), and extract new entities, relations, and events from raw unstructured text during KG construction.",
    "key_facts": [
      "rich textual descriptions",
      "generative link prediction (KGC)",
      "extracting entities and relations from unstructured text",
      "inductive generalization for unseen entities"
    ]
  },
  {
    "id": "GQ_05",
    "category": "Global",
    "question": "What are the future research directions and open challenges identified for unifying LLMs and KGs in Section 7?",
    "ground_truth_nodes": [
      "0016",
      "0017"
    ],
    "ground_truth_pages": [
      20,
      21
    ],
    "ground_truth_answer": "The key future directions include: 1) KGs for Hallucination Detection in LLMs, 2) Knowledge Editing using KGs, 3) Multimodal Knowledge Graph and LLM Unification, 4) Complex Multi-agent Reasoning and Synergy, and 5) Continual and Lifelong Knowledge Learning.",
    "key_facts": [
      "Hallucination Detection",
      "Knowledge Editing",
      "Multimodal Unification",
      "Complex Reasoning",
      "Continual Knowledge Learning"
    ]
  },
  {
    "id": "GQ_06",
    "category": "Global",
    "question": "Why are both implicit knowledge in LLMs and explicit knowledge in KGs necessary for reliable AI according to the authors?",
    "ground_truth_nodes": [
      "0001",
      "0003",
      "0015",
      "0020"
    ],
    "ground_truth_pages": [
      1,
      5,
      18,
      27
    ],
    "ground_truth_answer": "Implicit knowledge in neural parameters offers linguistic flexibility and intuitive common-sense reasoning, but lacks verifiability and precision. Explicit symbolic knowledge in KGs provides deterministic, verifiable, and editable facts. Unifying both produces reliable, grounded, and interpretable AI.",
    "key_facts": [
      "implicit knowledge provides linguistic flexibility and broad reasoning",
      "explicit knowledge provides deterministic, verifiable facts",
      "unification prevents hallucinations and adds grounding"
    ]
  },
  {
    "id": "GQ_07",
    "category": "Global",
    "question": "How does the taxonomy differentiate between one-way enhancement and two-way unification?",
    "ground_truth_nodes": [
      "0003",
      "0004",
      "0009",
      "0015"
    ],
    "ground_truth_pages": [
      5,
      7,
      11,
      18
    ],
    "ground_truth_answer": "One-way enhancement consists of either KG-enhanced LLMs (KG as an auxiliary feature to improve an LLM) or LLM-augmented KGs (LLM as a tool to complete or build a KG). Two-way unification (Synergized LLMs + KGs) represents a mutually reciprocal architecture where both representations and reasoning processes co-evolve.",
    "key_facts": [
      "one-way: KG-enhanced LLMs or LLM-augmented KGs",
      "two-way: Synergized LLMs + KGs",
      "reciprocal architecture with mutual representation and reasoning"
    ]
  },
  {
    "id": "GQ_08",
    "category": "Global",
    "question": "What role do prompt engineering and in-context learning play across both KG-enhanced LLMs and LLM-augmented KGs?",
    "ground_truth_nodes": [
      "0002",
      "0006",
      "0011",
      "0014"
    ],
    "ground_truth_pages": [
      3,
      9,
      13,
      17
    ],
    "ground_truth_answer": "In KG-enhanced LLMs, prompt engineering linearizes retrieved graph subgraphs into text prompts for in-context grounding. In LLM-augmented KGs, in-context demonstrations enable LLMs to perform link prediction, triple verbalization, and SPARQL generation without task-specific fine-tuning.",
    "key_facts": [
      "linearizing graph triples into prompts for grounding",
      "few-shot demonstration for link prediction",
      "zero-shot/few-shot SPARQL parsing without fine-tuning"
    ]
  },
  {
    "id": "GQ_09",
    "category": "Global",
    "question": "How does the paper classify Knowledge Graphs based on knowledge scope across the surveyed literature?",
    "ground_truth_nodes": [
      "0002",
      "0003"
    ],
    "ground_truth_pages": [
      3,
      4,
      5
    ],
    "ground_truth_answer": "KGs are classified by scope into: Encyclopedia KGs (general world facts like Wikidata), Commonsense KGs (everyday concepts and social rules like ConceptNet), Domain-specific KGs (specialized medical or financial graphs like UMLS), and Multimodal KGs (incorporating visual/audio entities).",
    "key_facts": [
      "Encyclopedia KGs",
      "Commonsense KGs",
      "Domain-specific KGs",
      "Multimodal KGs"
    ]
  },
  {
    "id": "GQ_10",
    "category": "Global",
    "question": "What are the primary trade-offs between static knowledge integration (pre-training) versus dynamic knowledge integration (inference)?",
    "ground_truth_nodes": [
      "0005",
      "0006",
      "0007"
    ],
    "ground_truth_pages": [
      7,
      9,
      10
    ],
    "ground_truth_answer": "Pre-training integrates knowledge deeply into model weights with zero retrieval latency at test time, but requires expensive retraining to update facts. Inference-stage integration is dynamic and requires no retraining, but introduces retrieval overhead, latency, and context window constraints.",
    "key_facts": [
      "pre-training: deep weight fusion, zero inference retrieval overhead, expensive retraining for updates",
      "inference: dynamic updates without retraining, higher latency, context window limits"
    ]
  },
  {
    "id": "GQ_11",
    "category": "Global",
    "question": "How does the survey structure the role of LLMs across the full lifecycle of Knowledge Graph curation?",
    "ground_truth_nodes": [
      "0009",
      "0012"
    ],
    "ground_truth_pages": [
      11,
      14,
      15
    ],
    "ground_truth_answer": "LLMs participate in: 1) Initial construction from raw corpora via entity, relation, and event extraction, 2) Entity linking and canonicalization, 3) Knowledge Graph Completion for missing links, and 4) Downstream KGQA and text verbalization.",
    "key_facts": [
      "entity, relation, and event extraction",
      "entity linking",
      "graph completion (KGC)",
      "downstream KGQA and verbalization"
    ]
  },
  {
    "id": "GQ_12",
    "category": "Global",
    "question": "What are common benchmark datasets mentioned throughout the survey for evaluating LLM and KG methods?",
    "ground_truth_nodes": [
      "0005",
      "0008",
      "0010",
      "0011",
      "0014"
    ],
    "ground_truth_pages": [
      7,
      10,
      11,
      12,
      17
    ],
    "ground_truth_answer": "Frequently cited benchmarks include: LAMA (knowledge probing), FB15k-237 and WN18RR (KG embedding and completion), WebQuestionsSP and ComplexWebQuestions (KGQA), ConceptNet and ATOMIC (commonsense reasoning), and CoNLL/TACRED (information extraction).",
    "key_facts": [
      "LAMA",
      "FB15k-237",
      "WN18RR",
      "WebQuestionsSP",
      "ConceptNet/ATOMIC"
    ]
  },
  {
    "id": "GQ_13",
    "category": "Global",
    "question": "How does synergized reasoning in Section 6 go beyond standard Retrieval-Augmented Generation (RAG)?",
    "ground_truth_nodes": [
      "0006",
      "0015"
    ],
    "ground_truth_pages": [
      9,
      18,
      19
    ],
    "ground_truth_answer": "Standard RAG uses one-way linear text retrieval to supply context to an LLM. Synergized reasoning involves iterative, multi-hop symbolic path searching, where the LLM guides graph traversal and symbolic algorithms constrain and verify each step of model inference.",
    "key_facts": [
      "standard RAG is one-way linear retrieval",
      "synergized reasoning is multi-hop iterative traversal",
      "LLM guides traversal while symbolic graph verifies steps"
    ]
  },
  {
    "id": "GQ_14",
    "category": "Global",
    "question": "What are the ethical, hallucination, and factual accuracy implications discussed in the roadmap?",
    "ground_truth_nodes": [
      "0001",
      "0004",
      "0016"
    ],
    "ground_truth_pages": [
      1,
      7,
      20
    ],
    "ground_truth_answer": "Hallucinations pose serious safety risks in high-stakes domains (healthcare, law). KGs offer a verifiable, traceable audit trail for model claims, enabling factual verification, reduced toxicity, and reliable hallucination detection.",
    "key_facts": [
      "hallucination safety risks in high-stakes domains",
      "KGs provide traceable audit trail and verifiable facts",
      "reducing toxicity and increasing trustworthiness"
    ]
  },
  {
    "id": "GQ_15",
    "category": "Global",
    "question": "How do multimodal aspects intersect with LLMs and KGs in the future vision of the paper?",
    "ground_truth_nodes": [
      "0002",
      "0016"
    ],
    "ground_truth_pages": [
      4,
      20
    ],
    "ground_truth_answer": "The roadmap envisions unifying vision-language models with multimodal knowledge graphs (MMKGs), allowing grounding of both visual tokens and textual entities in structured multimodal knowledge for cross-modal reasoning.",
    "key_facts": [
      "vision-language models combined with MMKGs",
      "grounding visual and textual entities in structured knowledge",
      "cross-modal reasoning"
    ]
  },
  {
    "id": "GQ_16",
    "category": "Global",
    "question": "How has the evolution of pre-trained models from BERT to auto-regressive models (GPT-3/4) affected KG integration strategies?",
    "ground_truth_nodes": [
      "0001",
      "0002",
      "0005",
      "0006"
    ],
    "ground_truth_pages": [
      1,
      3,
      7,
      9
    ],
    "ground_truth_answer": "Early models (BERT, RoBERTa) integrated KGs primarily via pre-training objective modification and embedding fusion encoders. Modern billion-parameter models shift toward inference-time prompting, in-context retrieval (RAG), and agentic tool-use, avoiding massive retraining costs.",
    "key_facts": [
      "BERT/RoBERTa focused on pre-training objectives and weight fusion",
      "modern LLMs shift to prompt-based retrieval and in-context RAG",
      "avoiding expensive retraining of 100B+ parameter models"
    ]
  },
  {
    "id": "GQ_17",
    "category": "Global",
    "question": "What are the core limitations of current KG-to-text generation systems identified in Section 5.4?",
    "ground_truth_nodes": [
      "0013"
    ],
    "ground_truth_pages": [
      16
    ],
    "ground_truth_answer": "Core limitations include hallucinating facts not grounded in the input subgraph, difficulty linearizing complex cyclic graph topologies into sequential natural text, and handling out-of-vocabulary entities.",
    "key_facts": [
      "hallucinating non-grounded facts",
      "linearizing complex cyclic topologies",
      "out-of-vocabulary entities"
    ]
  },
  {
    "id": "GQ_18",
    "category": "Global",
    "question": "How does the survey summarize the transition from heuristic KG embeddings to LLM-driven embeddings in Section 5.1?",
    "ground_truth_nodes": [
      "0010"
    ],
    "ground_truth_pages": [
      11,
      12
    ],
    "ground_truth_answer": "Early KGE methods (TransE, RotatE) relied solely on graph topology and distance heuristics, failing on sparse or unseen entities. LLM-augmented KGE incorporates rich linguistic entity descriptions, allowing semantic generalization and zero-shot entity representation.",
    "key_facts": [
      "traditional KGE relied only on graph topology heuristics",
      "LLM-driven KGE utilizes rich textual entity descriptions",
      "handles sparse graphs and zero-shot entity representation"
    ]
  },
  {
    "id": "GQ_19",
    "category": "Global",
    "question": "What is the roadmap's vision for continuous knowledge updating without catastrophic forgetting?",
    "ground_truth_nodes": [
      "0006",
      "0007",
      "0016"
    ],
    "ground_truth_pages": [
      9,
      10,
      20
    ],
    "ground_truth_answer": "The roadmap envisions combining dynamic retrieval-based inference with model editing techniques (Locating and Editing or ROME), using KGs to pinpoint outdated factual parameters and update them surgical-style without degrading general language capabilities.",
    "key_facts": [
      "model editing / surgical parameter updates",
      "dynamic retrieval-based inference",
      "preventing catastrophic forgetting of general language abilities"
    ]
  },
  {
    "id": "GQ_20",
    "category": "Global",
    "question": "What is the final conclusion of the survey regarding unified foundation models and structured knowledge?",
    "ground_truth_nodes": [
      "0017"
    ],
    "ground_truth_pages": [
      21
    ],
    "ground_truth_answer": "Section 8 concludes that LLMs and KGs are inherently complementary: neither can solve general intelligence alone. The future of AI lies in deep, synergized unification combining inductive statistical learning with deductive symbolic knowledge.",
    "key_facts": [
      "inherently complementary paradigms",
      "neither LLM nor KG alone is sufficient",
      "combines inductive statistical learning with deductive symbolic knowledge"
    ]
  },
  {
    "id": "MH_01",
    "category": "Multi-hop",
    "question": "How does the knowledge retrieval step in Section 4.2 relate to the KGQA information retrieval paradigm in Section 5.5?",
    "ground_truth_nodes": [
      "0006",
      "0014"
    ],
    "ground_truth_pages": [
      9,
      17
    ],
    "ground_truth_answer": "Both mechanisms perform entity linking on user queries to retrieve relevant subgraphs from a KG; in Section 4.2 the retrieved subgraph is converted into prompt context for text generation, while in Section 5.5 the candidate subgraph is queried to isolate the exact answer entity.",
    "key_facts": [
      "both perform entity linking to retrieve relevant subgraphs",
      "Section 4.2 uses subgraphs as prompt context for LLMs",
      "Section 5.5 queries the subgraph to predict the target answer entity"
    ]
  },
  {
    "id": "MH_02",
    "category": "Multi-hop",
    "question": "Compare the computational overhead of KG-enhanced pre-training in Section 4.1 with the inference latency trade-offs analyzed in Section 4.3.",
    "ground_truth_nodes": [
      "0005",
      "0007"
    ],
    "ground_truth_pages": [
      7,
      9,
      10
    ],
    "ground_truth_answer": "Section 4.1 pre-training requires massive upfront computational resources (training on billions of tokens with multi-objective masking), but yields zero extra latency during runtime. Section 4.3 notes that inference methods have low upfront cost but add runtime latency due to entity linking and graph retrieval per query.",
    "key_facts": [
      "pre-training has huge upfront compute cost but fast runtime",
      "inference has zero retraining cost but adds per-query latency",
      "Section 4.3 compares runtime vs training trade-offs"
    ]
  },
  {
    "id": "MH_03",
    "category": "Multi-hop",
    "question": "How do the entity masking objectives in Section 4.1 compare with the entity discovery and typing methods in Section 5.3?",
    "ground_truth_nodes": [
      "0005",
      "0012"
    ],
    "ground_truth_pages": [
      8,
      14
    ],
    "ground_truth_answer": "Entity masking in Section 4.1 uses known KG entities to mask tokens so the language model learns world facts; in contrast, Section 5.3 entity discovery applies the trained LLM in reverse to identify, type, and link new unknown entity mentions from raw text into a KG.",
    "key_facts": [
      "Section 4.1 uses existing KG entities to mask tokens for model training",
      "Section 5.3 uses LLMs to extract and discover new entities from raw text",
      "inverse relationship between knowledge consumption and knowledge creation"
    ]
  },
  {
    "id": "MH_04",
    "category": "Multi-hop",
    "question": "How do the limitations of KGs in Appendix A (such as incompleteness and unseen facts) motivate the LLM-augmented KGC methods in Section 5.2?",
    "ground_truth_nodes": [
      "0011",
      "0020"
    ],
    "ground_truth_pages": [
      12,
      28
    ],
    "ground_truth_answer": "Appendix A notes KGs suffer from severe incompleteness and fail on unseen facts because they ignore text. Section 5.2 directly solves this by using LLMs to read textual entity descriptions and generate missing links (PaD) or score unseen candidate entities (PaE).",
    "key_facts": [
      "Appendix A highlights incompleteness and unseen facts",
      "Section 5.2 KGC uses LLMs to predict missing triples",
      "LLM text understanding overcomes symbolic structural sparsity"
    ]
  },
  {
    "id": "MH_05",
    "category": "Multi-hop",
    "question": "How does the 'Black-box' disadvantage of LLMs in Appendix A connect to the KG-based interpretability techniques in Section 4.4?",
    "ground_truth_nodes": [
      "0008",
      "0020"
    ],
    "ground_truth_pages": [
      10,
      27
    ],
    "ground_truth_answer": "Appendix A highlights LLMs as opaque black boxes whose reasoning cannot be verified. Section 4.4 resolves this by utilizing KGs as explicit conceptual maps to probe which facts the LLM actually knows and trace attention weights to explain decisions.",
    "key_facts": [
      "Appendix A identifies black-box lack of interpretability",
      "Section 4.4 uses KGs as structured probes and validation maps",
      "allows inspecting internal knowledge and tracing reasoning paths"
    ]
  },
  {
    "id": "MH_06",
    "category": "Multi-hop",
    "question": "In what ways do Commonsense KGs defined in Section 2 get utilized in both Pre-training (Section 4.1) and Reasoning (Section 6)?",
    "ground_truth_nodes": [
      "0002",
      "0005",
      "0015"
    ],
    "ground_truth_pages": [
      4,
      8,
      19
    ],
    "ground_truth_answer": "In Section 2, Commonsense KGs (ConceptNet, ATOMIC) capture everyday physical/social relations. In Section 4.1, they provide relational paths for entity-guided pre-training. In Section 6, they provide symbolic reasoning chains that guide LLMs through multi-hop commonsense question answering.",
    "key_facts": [
      "Section 2 defines ConceptNet/ATOMIC",
      "Section 4.1 uses them for pre-training entity objectives",
      "Section 6 uses them for symbolic multi-hop reasoning chains"
    ]
  },
  {
    "id": "MH_07",
    "category": "Multi-hop",
    "question": "How does Knowledge Graph Embedding in Section 5.1 provide representations that can be used for KG-enhanced inference in Section 4.2?",
    "ground_truth_nodes": [
      "0006",
      "0010"
    ],
    "ground_truth_pages": [
      9,
      11
    ],
    "ground_truth_answer": "Section 5.1 methods generate continuous entity/relation vector representations; Section 4.2 takes these low-dimensional embeddings and feeds them into the LLM's cross-attention or soft-prompt layers during inference for factual steering.",
    "key_facts": [
      "Section 5.1 maps entities/relations to vector embeddings",
      "Section 4.2 injects these embeddings into LLM attention or soft prompts",
      "bridges symbolic triples and continuous latent spaces"
    ]
  },
  {
    "id": "MH_08",
    "category": "Multi-hop",
    "question": "How does the hallucination drawback of LLMs mentioned in Section 1 link to the future hallucination detection milestone in Section 7.1?",
    "ground_truth_nodes": [
      "0001",
      "0016"
    ],
    "ground_truth_pages": [
      2,
      20
    ],
    "ground_truth_answer": "Section 1 identifies hallucination (generating factually false content) as a critical barrier to deploying LLMs. Section 7.1 establishes the research milestone of using KGs as an automated verification oracle to check LLM outputs and eliminate hallucinations.",
    "key_facts": [
      "Section 1 defines hallucination as a core vulnerability",
      "Section 7.1 establishes KGs as verification oracles",
      "detecting and mitigating false model assertions"
    ]
  },
  {
    "id": "MH_09",
    "category": "Multi-hop",
    "question": "Compare how textual information is incorporated in KGC (Section 5.2) versus KG-to-text generation (Section 5.4).",
    "ground_truth_nodes": [
      "0011",
      "0013"
    ],
    "ground_truth_pages": [
      12,
      16
    ],
    "ground_truth_answer": "In KGC (5.2), text descriptions are used as input features to predict missing graph links (output is a graph triple/entity). In KG-to-text (5.4), the input is a graph subgraph and the output is a coherent natural language paragraph describing the graph.",
    "key_facts": [
      "Section 5.2 uses text as input to predict graph links",
      "Section 5.4 uses graph as input to generate fluent text",
      "inverse modality translation"
    ]
  },
  {
    "id": "MH_10",
    "category": "Multi-hop",
    "question": "How does the PaD (Pre-trained as Decoders) framework in Section 5.2 relate to prompt-based KGQA in Section 5.5?",
    "ground_truth_nodes": [
      "0011",
      "0014"
    ],
    "ground_truth_pages": [
      13,
      17
    ],
    "ground_truth_answer": "Both formulate the task as auto-regressive text generation via prompting: PaD prompts the LLM to generate the missing entity name for a triple (head, relation, ?), while prompt-based KGQA prompts the LLM to generate the natural answer to a user query given retrieved triples.",
    "key_facts": [
      "both use auto-regressive prompting",
      "PaD prompts for missing entity tokens",
      "KGQA prompts for natural question answers"
    ]
  },
  {
    "id": "MH_11",
    "category": "Multi-hop",
    "question": "How do structural triples in Section 2 contrast with unstructured corpora in Section 1, and how does Section 3 propose bridging them?",
    "ground_truth_nodes": [
      "0001",
      "0002",
      "0003"
    ],
    "ground_truth_pages": [
      1,
      3,
      5
    ],
    "ground_truth_answer": "Section 1 describes unstructured corpora as continuous text with rich semantics but implicit facts; Section 2 describes KGs as discrete (head, relation, tail) triples. Section 3's roadmap bridges them via unified representations, dynamic retrieval, and synergized reasoning.",
    "key_facts": [
      "unstructured corpus has continuous semantics but implicit facts",
      "KGs have discrete structured triples",
      "Section 3 roadmap unifies them via representations, retrieval, and synergy"
    ]
  },
  {
    "id": "MH_12",
    "category": "Multi-hop",
    "question": "How does domain-specific KG construction in Section 5.3 alleviate the domain-specific knowledge gap of LLMs noted in Appendix A?",
    "ground_truth_nodes": [
      "0012",
      "0020"
    ],
    "ground_truth_pages": [
      14,
      27
    ],
    "ground_truth_answer": "Appendix A notes general LLMs lack domain-specific expertise. Section 5.3 enables building domain-specific KGs from proprietary medical or legal texts via relation extraction, which can then be dynamically queried by LLMs during inference.",
    "key_facts": [
      "Appendix A notes LLMs lack domain-specific expertise",
      "Section 5.3 constructs domain KGs from specialized text",
      "provides accurate domain facts for LLMs"
    ]
  },
  {
    "id": "MH_13",
    "category": "Multi-hop",
    "question": "Contrast the role of LLMs as feature encoders in KGE (Section 5.1) with LLMs as cognitive processors in Synergized frameworks (Section 6).",
    "ground_truth_nodes": [
      "0010",
      "0015"
    ],
    "ground_truth_pages": [
      11,
      18
    ],
    "ground_truth_answer": "In Section 5.1, the LLM is merely a static feature extractor that converts entity descriptions into vectors. In Section 6, the LLM is an active cognitive agent that iteratively reasons, plans graph traversals, and interacts bidirectionally with the KG.",
    "key_facts": [
      "Section 5.1: passive vector feature extractor",
      "Section 6: active cognitive agent planning and reasoning",
      "bidirectional synergy"
    ]
  },
  {
    "id": "MH_14",
    "category": "Multi-hop",
    "question": "How does dynamic knowledge fusion in Section 4.2 compare with synergized bidirectional knowledge representation in Section 6?",
    "ground_truth_nodes": [
      "0006",
      "0015"
    ],
    "ground_truth_pages": [
      9,
      18
    ],
    "ground_truth_answer": "Dynamic fusion in 4.2 is a one-way pipeline (retrieve KG subgraph, feed into LLM prompt or attention). Synergized representation in Section 6 trains a joint multimodal graph-text model where token embeddings and graph node embeddings inhabit a shared semantic space.",
    "key_facts": [
      "Section 4.2 is one-way pipeline injection",
      "Section 6 is joint dual-modality unified embedding space",
      "bidirectional mutual representation"
    ]
  },
  {
    "id": "MH_15",
    "category": "Multi-hop",
    "question": "How do relation extraction pipelines in Section 5.3 feed into the inductive completion methods of Section 5.2?",
    "ground_truth_nodes": [
      "0011",
      "0012"
    ],
    "ground_truth_pages": [
      13,
      14
    ],
    "ground_truth_answer": "Section 5.3 extracts new relations and entity mentions from unstructured text to build initial graph edges; Section 5.2 then takes these newly expanded subgraphs and performs inductive relation prediction to infer missing links on the newly extracted nodes.",
    "key_facts": [
      "Section 5.3 extracts relations from text",
      "Section 5.2 predicts missing links on the extracted entities",
      "end-to-end pipeline from raw text to complete KG"
    ]
  },
  {
    "id": "MH_16",
    "category": "Multi-hop",
    "question": "How does the evaluation of knowledge probing in Section 4.4 compare with knowledge graph completion benchmarks in Section 5.2?",
    "ground_truth_nodes": [
      "0008",
      "0011"
    ],
    "ground_truth_pages": [
      10,
      12
    ],
    "ground_truth_answer": "Probing (4.4) tests an LLM's frozen parametric memory on cloze prompts without graph structure at test time. KGC (5.2) evaluates graph link prediction by scoring candidate entities using both graph topology and text descriptions.",
    "key_facts": [
      "probing evaluates frozen internal parameters with cloze text",
      "KGC evaluates graph link prediction using topology and text",
      "tests parametric memory vs graph completion capability"
    ]
  },
  {
    "id": "MH_17",
    "category": "Multi-hop",
    "question": "How does the distinction between Encyclopedia KGs and Domain-specific KGs in Section 2 impact their integration in pre-training (Section 4.1)?",
    "ground_truth_nodes": [
      "0002",
      "0005"
    ],
    "ground_truth_pages": [
      3,
      7
    ],
    "ground_truth_answer": "Encyclopedia KGs (Wikidata) are integrated into general-domain pre-training to boost common factual accuracy across broad benchmarks. Domain-specific KGs (UMLS) are integrated into specialized pre-trained models (e.g. BioBERT) to master technical medical/scientific vocabularies.",
    "key_facts": [
      "Encyclopedia KGs enhance general pre-training (broad benchmarks)",
      "Domain-specific KGs enhance specialized domain pre-training (BioBERT/UMLS)",
      "aligns knowledge scope with model target"
    ]
  },
  {
    "id": "MH_18",
    "category": "Multi-hop",
    "question": "How do the reasoning algorithms in KGs (Appendix A) complement the probabilistic generation of LLMs in Synergized Reasoning (Section 6)?",
    "ground_truth_nodes": [
      "0015",
      "0020"
    ],
    "ground_truth_pages": [
      19,
      28
    ],
    "ground_truth_answer": "Appendix A notes KGs have deterministic, decisive, and interpretable symbolic reasoning algorithms, contrasting with probabilistic LLM sampling. Section 6 marries both: the LLM provides flexible natural language reasoning, while the KG's symbolic path search guarantees factual decisiveness and eliminates hallucinated steps.",
    "key_facts": [
      "Appendix A: KGs have deterministic symbolic reasoning",
      "LLMs have probabilistic sampling",
      "Section 6 combines them: LLM proposes hypotheses, KG guarantees factual correctness"
    ]
  },
  {
    "id": "MH_19",
    "category": "Multi-hop",
    "question": "How do prompt engineering techniques discussed in Section 2 enable zero-shot relation extraction in Section 5.3?",
    "ground_truth_nodes": [
      "0002",
      "0012"
    ],
    "ground_truth_pages": [
      3,
      15
    ],
    "ground_truth_answer": "Section 2 explains that prompt engineering formats downstream tasks into natural language prompts; Section 5.3 utilizes this by formulating relation extraction as a question-answering or masked completion prompt (e.g., 'What is the relationship between X and Y?'), enabling zero-shot relation extraction without labeled training data.",
    "key_facts": [
      "Section 2 prompt engineering reformulates NLP tasks",
      "Section 5.3 formats relation extraction as QA or cloze prompts",
      "enables zero-shot extraction without supervised training"
    ]
  },
  {
    "id": "MH_20",
    "category": "Multi-hop",
    "question": "Why does knowledge obsolescence discussed in Section 4.3 motivate the future direction of Knowledge Editing in Section 7?",
    "ground_truth_nodes": [
      "0007",
      "0016"
    ],
    "ground_truth_pages": [
      9,
      20
    ],
    "ground_truth_answer": "Section 4.3 highlights that real-world knowledge changes constantly, making static pre-trained models outdated. Section 7 identifies Knowledge Editing as the required future breakthrough to locate and alter specific factual weights in an LLM when KG facts update.",
    "key_facts": [
      "Section 4.3 emphasizes real-world knowledge evolves continuously",
      "static pre-training suffers from knowledge obsolescence",
      "Section 7 proposes Knowledge Editing to surgically update outdated factual weights"
    ]
  },
  {
    "id": "SQ_01",
    "category": "Summary",
    "question": "Provide a comprehensive summary of the 'KG-enhanced LLMs' framework presented in Section 4.",
    "ground_truth_nodes": [
      "0004",
      "0005",
      "0006",
      "0007",
      "0008"
    ],
    "ground_truth_pages": [
      7,
      8,
      9,
      10
    ],
    "ground_truth_answer": "KG-enhanced LLMs integrate structured knowledge into language models across three stages: 1) Pre-training (incorporating entity embeddings and knowledge-guided objectives like entity masking), 2) Inference (dynamically retrieving relevant subgraphs and injecting them via prompt templates or visibility attention without retraining), and 3) Interpretability (using KGs to probe internal factual knowledge and trace decision pathways).",
    "key_facts": [
      "integrates KGs into LLMs",
      "Pre-training: entity embeddings and masking",
      "Inference: dynamic retrieval and prompting without retraining",
      "Interpretability: knowledge probing and decision explanation"
    ]
  },
  {
    "id": "SQ_02",
    "category": "Summary",
    "question": "Provide a comprehensive summary of the 'LLM-augmented KGs' framework presented in Section 5.",
    "ground_truth_nodes": [
      "0009",
      "0010",
      "0011",
      "0012",
      "0013",
      "0014"
    ],
    "ground_truth_pages": [
      11,
      12,
      14,
      16,
      17
    ],
    "ground_truth_answer": "LLM-augmented KGs leverages LLMs to enhance structural knowledge graphs across five core areas: 1) KG Embedding (using LLMs as text encoders for entities), 2) KG Completion (inferring missing links via PaE or PaD generative decoding), 3) KG Construction (discovering entities, relations, and events from raw text), 4) KG-to-text generation (verbalizing subgraphs into fluent natural language), and 5) KGQA (semantic parsing and information retrieval over graphs).",
    "key_facts": [
      "KG Embedding (KGE)",
      "KG Completion (KGC)",
      "KG Construction (NER, RE, EE)",
      "KG-to-text generation",
      "KG Question Answering (KGQA)"
    ]
  },
  {
    "id": "SQ_03",
    "category": "Summary",
    "question": "Summarize the 'Synergized LLMs + KGs' framework and its two main paradigms in Section 6.",
    "ground_truth_nodes": [
      "0015"
    ],
    "ground_truth_pages": [
      18,
      19
    ],
    "ground_truth_answer": "Synergized LLMs + KGs represents bidirectional unification with two main paradigms: 1) Synergized Knowledge Representation, where models jointly learn unified representations of text and graphs in a shared semantic space, and 2) Synergized Knowledge Reasoning, where LLMs act as intuitive cognitive reasoning engines and KGs act as symbolic, verifiable knowledge anchors for multi-hop deduction.",
    "key_facts": [
      "bidirectional unification",
      "Synergized Knowledge Representation: joint unified embeddings",
      "Synergized Knowledge Reasoning: LLM reasoning guided by KG symbolic verification"
    ]
  },
  {
    "id": "SQ_04",
    "category": "Summary",
    "question": "Summarize the Roadmap and categorization scheme introduced in Section 3.",
    "ground_truth_nodes": [
      "0003"
    ],
    "ground_truth_pages": [
      5,
      6
    ],
    "ground_truth_answer": "Section 3 organizes the entire landscape into a systematic roadmap: 1) KG-enhanced LLMs (pre-training, inference, interpretability), 2) LLM-augmented KGs (embedding, completion, construction, KG-to-text, KGQA), and 3) Synergized LLMs + KGs (representation and reasoning), mapping how models evolve from one-way enhancement to full synergy.",
    "key_facts": [
      "three primary frameworks",
      "systematic mapping of sub-tasks",
      "progression from one-way enhancement to mutual synergy"
    ]
  },
  {
    "id": "SQ_05",
    "category": "Summary",
    "question": "Summarize the comparison between KG-enhanced Pre-training and Inference in Section 4.3.",
    "ground_truth_nodes": [
      "0007"
    ],
    "ground_truth_pages": [
      9,
      10
    ],
    "ground_truth_answer": "Pre-training aligns knowledge deeply into model weights with zero test-time retrieval overhead, but is computationally expensive and cannot be easily updated. Inference methods require no retraining and support real-time dynamic knowledge updates, but introduce inference latency, retrieval complexity, and context window bottlenecks.",
    "key_facts": [
      "pre-training: deep alignment, zero test retrieval overhead, rigid updates, high cost",
      "inference: dynamic knowledge updates, no retraining, higher latency, context length limits"
    ]
  },
  {
    "id": "SQ_06",
    "category": "Summary",
    "question": "Summarize the sub-tasks and LLM roles in Knowledge Graph Construction (Section 5.3).",
    "ground_truth_nodes": [
      "0012"
    ],
    "ground_truth_pages": [
      14,
      15
    ],
    "ground_truth_answer": "KG Construction covers three sub-tasks: 1) Entity Discovery (NER, Entity Typing, and Entity Linking), 2) Relation Extraction (identifying predicates between entities), and 3) Event Extraction (identifying event triggers and arguments). LLMs excel in all three via prompt-based zero-shot and few-shot extraction from unstructured text.",
    "key_facts": [
      "Entity Discovery: NER, typing, linking",
      "Relation Extraction",
      "Event Extraction: triggers and arguments",
      "LLMs enable few-shot and zero-shot extraction"
    ]
  },
  {
    "id": "SQ_07",
    "category": "Summary",
    "question": "Summarize the pros and cons of LLMs and KGs as detailed in Appendix A.",
    "ground_truth_nodes": [
      "0020"
    ],
    "ground_truth_pages": [
      27,
      28
    ],
    "ground_truth_answer": "LLM pros: General Knowledge, Language Processing, Generalizability. LLM cons: Implicit Knowledge, Hallucination, Indecisiveness, Black-box nature, Lacking Domain Knowledge. KG pros: Structural Knowledge, Accuracy, Decisiveness, Interpretability, Domain Knowledge, Evolving Knowledge. KG cons: Incompleteness, Lacking Language Understanding, Unseen Facts.",
    "key_facts": [
      "LLM pros: general knowledge, language, generalizability",
      "LLM cons: hallucination, black-box, implicit, indecisive",
      "KG pros: structural, accurate, decisive, interpretable",
      "KG cons: incomplete, lacks language understanding, unseen facts"
    ]
  },
  {
    "id": "SQ_08",
    "category": "Summary",
    "question": "Summarize the key future directions and milestones outlined in Section 7.",
    "ground_truth_nodes": [
      "0016"
    ],
    "ground_truth_pages": [
      20,
      21
    ],
    "ground_truth_answer": "Section 7 highlights key frontiers: 1) Using KGs as ground truth to detect and eliminate LLM hallucinations, 2) Knowledge Editing to surgically update parameters, 3) Multimodal graph and vision-language unification, 4) Complex multi-hop reasoning agents, and 5) Lifelong continually updated unified AI systems.",
    "key_facts": [
      "Hallucination Detection",
      "Knowledge Editing",
      "Multimodal Unification",
      "Complex Multi-agent Reasoning",
      "Continual Knowledge Learning"
    ]
  },
  {
    "id": "SQ_09",
    "category": "Summary",
    "question": "Summarize the background and classification of Knowledge Graphs in Section 2.",
    "ground_truth_nodes": [
      "0002"
    ],
    "ground_truth_pages": [
      3,
      4
    ],
    "ground_truth_answer": "Section 2 defines KGs as structured directed graphs of entity nodes and relation edges (head, relation, tail). It classifies them into four categories: Encyclopedia KGs (broad world knowledge), Commonsense KGs (everyday concepts), Domain-specific KGs (curated expert fields), and Multimodal KGs (including image and audio assets).",
    "key_facts": [
      "triples: head, relation, tail",
      "Encyclopedia KGs",
      "Commonsense KGs",
      "Domain-specific KGs",
      "Multimodal KGs"
    ]
  },
  {
    "id": "SQ_10",
    "category": "Summary",
    "question": "Summarize the main conclusions of the entire paper presented in Section 8.",
    "ground_truth_nodes": [
      "0017"
    ],
    "ground_truth_pages": [
      21
    ],
    "ground_truth_answer": "Section 8 concludes that unifying LLMs and KGs bridges statistical connectionist AI and symbolic deterministic AI. The survey has provided a full roadmap, demonstrated how both paradigms address each other's weaknesses, and provided a structured foundation for future research.",
    "key_facts": [
      "bridges connectionist statistical learning and symbolic AI",
      "both paradigms overcome each other's inherent weaknesses",
      "provides systematic foundation for future research"
    ]
  },
  {
    "id": "NF_01",
    "category": "Numerical/Factual",
    "question": "Which specific research grants funded this paper according to the Acknowledgments section?",
    "ground_truth_nodes": [
      "0018"
    ],
    "ground_truth_pages": [
      21
    ],
    "ground_truth_answer": "This research was supported by the Australian Research Council (ARC) under grants FT210100097 and DP240101547, and the National Natural Science Foundation of China (NSFC) under grant 62120106008.",
    "key_facts": [
      "ARC grant FT210100097",
      "ARC grant DP240101547",
      "NSFC grant 62120106008"
    ]
  },
  {
    "id": "NF_02",
    "category": "Numerical/Factual",
    "question": "In Section 2, what are the four representative Encyclopedia Knowledge Graphs explicitly named in the text?",
    "ground_truth_nodes": [
      "0002"
    ],
    "ground_truth_pages": [
      3
    ],
    "ground_truth_answer": "The four explicitly named representative Encyclopedia KGs are: Wikidata, DBpedia, YAGO, and Freebase.",
    "key_facts": [
      "Wikidata",
      "DBpedia",
      "YAGO",
      "Freebase"
    ]
  },
  {
    "id": "NF_03",
    "category": "Numerical/Factual",
    "question": "According to Section 2, what are the two representative Commonsense Knowledge Graphs explicitly mentioned?",
    "ground_truth_nodes": [
      "0002"
    ],
    "ground_truth_pages": [
      4
    ],
    "ground_truth_answer": "The two explicitly mentioned Commonsense Knowledge Graphs are ConceptNet and ATOMIC.",
    "key_facts": [
      "ConceptNet",
      "ATOMIC"
    ]
  },
  {
    "id": "NF_04",
    "category": "Numerical/Factual",
    "question": "What is the exact paper title and who are the authors listed in Node 0000?",
    "ground_truth_nodes": [
      "0000"
    ],
    "ground_truth_pages": [
      1
    ],
    "ground_truth_answer": "Title: 'Unifying Large Language Models and Knowledge Graphs: A Roadmap'. Authors: Shirui Pan, Senior Member, IEEE; Linhao Luo; Yufei Wang; Chen Chen; Jiapu Wang; Xindong Wu, Fellow, IEEE.",
    "key_facts": [
      "Unifying Large Language Models and Knowledge Graphs: A Roadmap",
      "Shirui Pan",
      "Linhao Luo",
      "Yufei Wang",
      "Chen Chen",
      "Jiapu Wang",
      "Xindong Wu"
    ]
  },
  {
    "id": "NF_05",
    "category": "Numerical/Factual",
    "question": "In Section 5.1, what are the two main categories of LLM-augmented KGE methods?",
    "ground_truth_nodes": [
      "0010"
    ],
    "ground_truth_pages": [
      11,
      12
    ],
    "ground_truth_answer": "The two main categories are: 1) LLM as Entity/Relation Encoders, and 2) Joint Structural and Textual Embedding.",
    "key_facts": [
      "LLM as Entity/Relation Encoders",
      "Joint Structural and Textual Embedding"
    ]
  },
  {
    "id": "NF_06",
    "category": "Numerical/Factual",
    "question": "In Section 5.2, what do the abbreviations PaE and PaD stand for in Knowledge Graph Completion?",
    "ground_truth_nodes": [
      "0011"
    ],
    "ground_truth_pages": [
      12,
      13
    ],
    "ground_truth_answer": "PaE stands for 'Pre-trained as Encoders', and PaD stands for 'Pre-trained as Decoders'.",
    "key_facts": [
      "Pre-trained as Encoders (PaE)",
      "Pre-trained as Decoders (PaD)"
    ]
  },
  {
    "id": "NF_07",
    "category": "Numerical/Factual",
    "question": "In Section 4.1, what are the three distinct categories of KG-enhanced pre-training approaches?",
    "ground_truth_nodes": [
      "0005"
    ],
    "ground_truth_pages": [
      7,
      8
    ],
    "ground_truth_answer": "The three categories are: 1) KG-enhanced Training Corpus, 2) Knowledge-guided Training Objectives, and 3) KG-enhanced Architecture (or Knowledge-fusion Encoders).",
    "key_facts": [
      "KG-enhanced Training Corpus",
      "Knowledge-guided Training Objectives",
      "KG-enhanced Architecture / Fusion Encoders"
    ]
  },
  {
    "id": "NF_08",
    "category": "Numerical/Factual",
    "question": "According to Section 3.1 and Fig. 6, how many primary unified frameworks are identified in the roadmap?",
    "ground_truth_nodes": [
      "0003"
    ],
    "ground_truth_pages": [
      5
    ],
    "ground_truth_answer": "Three primary unified frameworks are identified: 1) KG-enhanced LLMs, 2) LLM-augmented KGs, and 3) Synergized LLMs + KGs.",
    "key_facts": [
      "three (3) frameworks",
      "KG-enhanced LLMs",
      "LLM-augmented KGs",
      "Synergized LLMs + KGs"
    ]
  },
  {
    "id": "NF_09",
    "category": "Numerical/Factual",
    "question": "In Section 5.3, what are the four specific subtasks identified under Entity Discovery for Knowledge Graph Construction?",
    "ground_truth_nodes": [
      "0012"
    ],
    "ground_truth_pages": [
      14
    ],
    "ground_truth_answer": "Entity Discovery comprises: 1) Named Entity Recognition (NER), 2) Entity Typing, and 3) Entity Linking (or Entity Alignment / Disambiguation).",
    "key_facts": [
      "Named Entity Recognition (NER)",
      "Entity Typing",
      "Entity Linking"
    ]
  },
  {
    "id": "NF_10",
    "category": "Numerical/Factual",
    "question": "In Section 6, what are the two main branches of Synergized LLMs + KGs?",
    "ground_truth_nodes": [
      "0015"
    ],
    "ground_truth_pages": [
      18,
      19
    ],
    "ground_truth_answer": "The two main branches are: 1) Synergized Knowledge Representation, and 2) Synergized Knowledge Reasoning.",
    "key_facts": [
      "Synergized Knowledge Representation",
      "Synergized Knowledge Reasoning"
    ]
  }
]