data/gold_dataset.json

[
  {
    "id": "dqn_01",
    "question": "What neural network architecture did DQN use to play Atari games, and what was the input representation?",
    "expected_arxiv_ids": ["1312.5602"],
    "expected_key_claim": "convolutional neural network with raw pixel input, specifically 84x84 grayscale frames with the last 4 frames stacked as input",
    "difficulty": "easy"
  },
  {
    "id": "dqn_02",
    "question": "What is experience replay in DQN and why does it improve training stability?",
    "expected_arxiv_ids": ["1312.5602"],
    "expected_key_claim": "stores agent transitions in a replay buffer and samples random minibatches to break temporal correlations between consecutive updates",
    "difficulty": "easy"
  },
  {
    "id": "dqn_03",
    "question": "What is the target network in DQN and what problem does it address?",
    "expected_arxiv_ids": ["1312.5602"],
    "expected_key_claim": "a separate network with periodically-copied weights used to compute stable TD targets, reducing oscillation caused by moving targets in standard Q-learning",
    "difficulty": "medium"
  },
  {
    "id": "ddqn_01",
    "question": "What overestimation problem in standard DQN does Double DQN address?",
    "expected_arxiv_ids": ["1509.06461"],
    "expected_key_claim": "DQN uses the same network for both action selection and value evaluation in the max operator, causing systematic overestimation of Q-values",
    "difficulty": "medium"
  },
  {
    "id": "ddqn_02",
    "question": "How does Double DQN decouple action selection from action evaluation?",
    "expected_arxiv_ids": ["1509.06461"],
    "expected_key_claim": "the online network selects the greedy action while the target network evaluates its value, preventing the same network from both choosing and scoring actions",
    "difficulty": "medium"
  },
  {
    "id": "dueling_01",
    "question": "What are the Value and Advantage streams in the Dueling Network architecture and how are they combined?",
    "expected_arxiv_ids": ["1511.06581"],
    "expected_key_claim": "shared convolutional encoder splits into V(s) stream and A(s,a) stream; combined as Q(s,a) = V(s) + A(s,a) minus mean advantage for identifiability",
    "difficulty": "medium"
  },
  {
    "id": "dueling_02",
    "question": "Why must the advantage stream be mean-subtracted in the Dueling DQN aggregation module?",
    "expected_arxiv_ids": ["1511.06581"],
    "expected_key_claim": "without centering, V and A are not uniquely identifiable from Q alone; subtracting the mean advantage forces V to be the state value and A to be relative advantages",
    "difficulty": "hard"
  },
  {
    "id": "rainbow_01",
    "question": "What six algorithmic improvements does Rainbow combine?",
    "expected_arxiv_ids": ["1710.02298"],
    "expected_key_claim": "double Q-learning, prioritized experience replay, dueling networks, multi-step returns, distributional RL (C51), and noisy nets for exploration",
    "difficulty": "easy"
  },
  {
    "id": "rainbow_02",
    "question": "What is distributional RL and how does Rainbow use it?",
    "expected_arxiv_ids": ["1710.02298"],
    "expected_key_claim": "instead of estimating expected return, distributional RL models the full probability distribution over returns; Rainbow uses C51 which represents this as a categorical distribution over fixed discrete atoms",
    "difficulty": "hard"
  },
  {
    "id": "rainbow_03",
    "question": "What is noisy networks (NoisyNets) and how does Rainbow use it for exploration?",
    "expected_arxiv_ids": ["1710.02298"],
    "expected_key_claim": "replaces epsilon-greedy exploration with learned stochastic weights in the network's linear layers; noise parameters are learned by gradient descent, enabling state-dependent exploration",
    "difficulty": "hard"
  },
  {
    "id": "trpo_01",
    "question": "What is the trust region constraint in TRPO and why is it necessary?",
    "expected_arxiv_ids": ["1502.05477"],
    "expected_key_claim": "constrains KL divergence between old and new policy to be below a threshold delta at each update, preventing destructively large policy changes that collapse performance",
    "difficulty": "medium"
  },
  {
    "id": "trpo_02",
    "question": "What optimization algorithm does TRPO use to solve the constrained policy update problem?",
    "expected_arxiv_ids": ["1502.05477"],
    "expected_key_claim": "conjugate gradient to compute the natural policy gradient direction, then a backtracking line search to find the largest step satisfying the KL constraint",
    "difficulty": "hard"
  },
  {
    "id": "trpo_03",
    "question": "What surrogate objective does TRPO optimize and how does it relate to the true policy objective?",
    "expected_arxiv_ids": ["1502.05477"],
    "expected_key_claim": "optimizes a surrogate objective using importance sampling ratios of new to old policy probabilities, which is a local approximation to the true objective guaranteed to be a lower bound within the trust region",
    "difficulty": "hard"
  },
  {
    "id": "gae_01",
    "question": "What does GAE stand for and what fundamental tradeoff does it manage?",
    "expected_arxiv_ids": ["1506.02438"],
    "expected_key_claim": "Generalized Advantage Estimation; trades off bias versus variance in advantage estimates for policy gradient methods using an exponential weighting parameter lambda",
    "difficulty": "easy"
  },
  {
    "id": "gae_02",
    "question": "How does the GAE lambda parameter interpolate between Monte Carlo and TD advantage estimates?",
    "expected_arxiv_ids": ["1506.02438"],
    "expected_key_claim": "lambda=0 reduces to single-step TD advantage (low variance, high bias); lambda=1 reduces to Monte Carlo return minus baseline (low bias, high variance); intermediate values exponentially weight n-step returns",
    "difficulty": "medium"
  },
  {
    "id": "a3c_01",
    "question": "How does A3C use asynchronous parallel workers to stabilize training?",
    "expected_arxiv_ids": ["1602.01783"],
    "expected_key_claim": "multiple actor-learner threads each interact with independent environment copies and asynchronously send gradient updates to a shared global network, decorrelating experience without requiring a replay buffer",
    "difficulty": "medium"
  },
  {
    "id": "a3c_02",
    "question": "What advantage function does A3C use in its policy gradient update?",
    "expected_arxiv_ids": ["1602.01783"],
    "expected_key_claim": "uses n-step returns as the target minus the value function V(s) as a baseline; the value network is trained simultaneously as a critic to estimate state values",
    "difficulty": "medium"
  },
  {
    "id": "a3c_03",
    "question": "How does A3C handle both discrete and continuous action spaces?",
    "expected_arxiv_ids": ["1602.01783"],
    "expected_key_claim": "for discrete actions uses softmax policy; for continuous actions outputs mean and variance of a Gaussian, sampling actions from it; the same asynchronous framework applies to both",
    "difficulty": "hard"
  },
  {
    "id": "ddpg_01",
    "question": "How does DDPG extend DQN to continuous action spaces?",
    "expected_arxiv_ids": ["1509.02971"],
    "expected_key_claim": "uses a deterministic actor network that outputs the exact continuous action and a critic network that evaluates Q(s,a); applies the deterministic policy gradient theorem rather than maximizing over a discrete action set",
    "difficulty": "medium"
  },
  {
    "id": "ddpg_02",
    "question": "What exploration strategy does DDPG use for continuous action spaces?",
    "expected_arxiv_ids": ["1509.02971"],
    "expected_key_claim": "adds temporally-correlated Ornstein-Uhlenbeck noise to the deterministic policy output during training, encouraging exploration with smooth, momentum-based noise rather than independent Gaussian noise",
    "difficulty": "medium"
  },
  {
    "id": "ddpg_03",
    "question": "How does DDPG update its target networks and why does this differ from DQN?",
    "expected_arxiv_ids": ["1509.02971"],
    "expected_key_claim": "uses soft Polyak averaging (target = tau*online + (1-tau)*target) at every step instead of hard periodic copying, providing slower and more stable target updates for continuous control",
    "difficulty": "medium"
  },
  {
    "id": "instructgpt_01",
    "question": "What are the three training stages in InstructGPT's RLHF pipeline?",
    "expected_arxiv_ids": ["2203.02155"],
    "expected_key_claim": "first supervised fine-tuning on human-written demonstrations, then reward model training from pairwise human preference comparisons, then PPO policy optimization against the reward model with KL penalty",
    "difficulty": "easy"
  },
  {
    "id": "instructgpt_02",
    "question": "What is the KL penalty term in InstructGPT's RL objective and why is it included?",
    "expected_arxiv_ids": ["2203.02155"],
    "expected_key_claim": "penalizes KL divergence between the RLHF policy and the original SFT policy to prevent reward hacking, maintain language quality, and avoid the policy collapsing to adversarial outputs that game the reward model",
    "difficulty": "medium"
  },
  {
    "id": "instructgpt_03",
    "question": "How does InstructGPT collect human preference data for training the reward model?",
    "expected_arxiv_ids": ["2203.02155"],
    "expected_key_claim": "human labelers rank multiple model completions for the same prompt; these pairwise orderings are converted into preference pairs used to train the reward model with a cross-entropy loss",
    "difficulty": "easy"
  },
  {
    "id": "dpo_01",
    "question": "What is the core insight of DPO that eliminates the need for explicit RL?",
    "expected_arxiv_ids": ["2305.18290"],
    "expected_key_claim": "shows that the optimal RLHF policy implies a closed-form reward expressible as log probability ratios between the policy and reference model; this allows directly optimizing the policy from preference data without training a separate reward model",
    "difficulty": "hard"
  },
  {
    "id": "dpo_02",
    "question": "What is the DPO training objective and what data does it require?",
    "expected_arxiv_ids": ["2305.18290"],
    "expected_key_claim": "binary cross-entropy loss on preference pairs where loss increases the likelihood of preferred completions relative to rejected ones, weighted by how much the policy deviates from the reference model; requires only (prompt, chosen, rejected) triples",
    "difficulty": "medium"
  },
  {
    "id": "cross_01",
    "question": "How do DQN and Double DQN differ in their Bellman target computation?",
    "expected_arxiv_ids": ["1312.5602", "1509.06461"],
    "expected_key_claim": "DQN uses the target network for both selecting and evaluating the greedy action; Double DQN uses the online network to select the action and the target network only to evaluate it, eliminating the maximization bias",
    "difficulty": "medium"
  },
  {
    "id": "cross_02",
    "question": "What architectural innovation separates Dueling DQN from standard DQN?",
    "expected_arxiv_ids": ["1511.06581", "1312.5602"],
    "expected_key_claim": "Dueling DQN splits the final layers into two streams estimating state value V(s) and state-action advantages A(s,a) separately, which standard DQN does not do",
    "difficulty": "easy"
  },
  {
    "id": "cross_03",
    "question": "How do InstructGPT and DPO differ in their approach to aligning language models with human preferences?",
    "expected_arxiv_ids": ["2203.02155", "2305.18290"],
    "expected_key_claim": "InstructGPT trains an explicit reward model then runs PPO against it; DPO directly optimizes the policy from preference data using a classification loss, eliminating the reward model and RL training loop entirely",
    "difficulty": "medium"
  },
  {
    "id": "cross_04",
    "question": "How do DDPG and A3C differ in their approach to continuous control problems?",
    "expected_arxiv_ids": ["1509.02971", "1602.01783"],
    "expected_key_claim": "DDPG is off-policy and uses a deterministic actor-critic with experience replay; A3C is on-policy and uses asynchronous parallel workers without a replay buffer; both can handle continuous actions",
    "difficulty": "hard"
  }
]