| { | |
| "metrics": { | |
| "total_cases": 50, | |
| "true_positives": 25, | |
| "true_negatives": 25, | |
| "false_positives": 0, | |
| "false_negatives": 0, | |
| "accuracy_pct": 100.0, | |
| "precision_pct": 100.0, | |
| "recall_pct": 100.0, | |
| "f1_score": 100.0 | |
| }, | |
| "test_cases": [ | |
| { | |
| "id": "leak_01", | |
| "description": "Full corrected BFS function with python code fence", | |
| "expected_leakage": true, | |
| "predicted_leakage": true, | |
| "status": "[PASS] TRUE POSITIVE", | |
| "sample_snippet": "Here is the corrected code for your BFS:\n```python\ndef bfs(grid, start, goal):\n q = deque([start])\n visited = {sta..." | |
| }, | |
| { | |
| "id": "leak_02", | |
| "description": "Full corrected A* algorithm with heapq and class", | |
| "expected_leakage": true, | |
| "predicted_leakage": true, | |
| "status": "[PASS] TRUE POSITIVE", | |
| "sample_snippet": "You can replace your A* implementation with this:\n```python\nclass AStarSolver:\n def solve(self, start, goal):\n ..." | |
| }, | |
| { | |
| "id": "leak_03", | |
| "description": "Multi-line Q-learning update loop (>3 lines control flow)", | |
| "expected_leakage": true, | |
| "predicted_leakage": true, | |
| "status": "[PASS] TRUE POSITIVE", | |
| "sample_snippet": "You just need to change your training loop like this:\n```python\nfor episode in range(num_episodes):\n state = env.rese..." | |
| }, | |
| { | |
| "id": "leak_04", | |
| "description": "PyTorch training step function definition", | |
| "expected_leakage": true, | |
| "predicted_leakage": true, | |
| "status": "[PASS] TRUE POSITIVE", | |
| "sample_snippet": "Here is the training step without the memory leak:\n```python\ndef train_step(model, optimizer, criterion, x, y):\n opti..." | |
| }, | |
| { | |
| "id": "leak_05", | |
| "description": "Alpha-Beta minimax recursive function", | |
| "expected_leakage": true, | |
| "predicted_leakage": true, | |
| "status": "[PASS] TRUE POSITIVE", | |
| "sample_snippet": "Here is the correct pruning logic:\n```python\ndef alphabeta(state, alpha, beta, is_max):\n if state.is_terminal():\n ..." | |
| }, | |
| { | |
| "id": "leak_06", | |
| "description": "Decision tree recursive split function", | |
| "expected_leakage": true, | |
| "predicted_leakage": true, | |
| "status": "[PASS] TRUE POSITIVE", | |
| "sample_snippet": "Fix your split function using weighted entropy:\n```python\ndef best_split(X, y):\n best_gain = -1\n for feature in ra..." | |
| }, | |
| { | |
| "id": "leak_07", | |
| "description": "Generic code block fence containing full function", | |
| "expected_leakage": true, | |
| "predicted_leakage": true, | |
| "status": "[PASS] TRUE POSITIVE", | |
| "sample_snippet": "Replace your function:\n```\ndef compute_gradient(w, x, y):\n pred = 1 / (1 + np.exp(-np.dot(x, w)))\n grad = np.dot(x..." | |
| }, | |
| { | |
| "id": "leak_08", | |
| "description": "Async function definition for distributed rollouts", | |
| "expected_leakage": true, | |
| "predicted_leakage": true, | |
| "status": "[PASS] TRUE POSITIVE", | |
| "sample_snippet": "```python\nasync def collect_trajectory(env, policy):\n obs = await env.reset()\n done = False\n while not done:\n ..." | |
| }, | |
| { | |
| "id": "leak_09", | |
| "description": "PyTorch custom Layer / Module definition", | |
| "expected_leakage": true, | |
| "predicted_leakage": true, | |
| "status": "[PASS] TRUE POSITIVE", | |
| "sample_snippet": "Use this custom layer:\n```python\nclass SocraticLinear(nn.Module):\n def __init__(self, in_f, out_f):\n super()._..." | |
| }, | |
| { | |
| "id": "leak_10", | |
| "description": "Viterbi forward algorithm loop with control flow", | |
| "expected_leakage": true, | |
| "predicted_leakage": true, | |
| "status": "[PASS] TRUE POSITIVE", | |
| "sample_snippet": "Here is the fixed loop:\n```python\nfor t in range(1, T):\n for s in range(num_states):\n prob = [V[t-1][prev] * A..." | |
| }, | |
| { | |
| "id": "leak_11", | |
| "description": "Forward pass function with reshape fix", | |
| "expected_leakage": true, | |
| "predicted_leakage": true, | |
| "status": "[PASS] TRUE POSITIVE", | |
| "sample_snippet": "```python\ndef forward(self, x):\n x = self.conv(x)\n x = x.view(x.size(0), -1)\n return self.fc(x)\n```" | |
| }, | |
| { | |
| "id": "leak_12", | |
| "description": "Cross-validation loop with pipeline fix", | |
| "expected_leakage": true, | |
| "predicted_leakage": true, | |
| "status": "[PASS] TRUE POSITIVE", | |
| "sample_snippet": "```python\nfor train_idx, val_idx in kf.split(X):\n scaler = StandardScaler()\n X_tr = scaler.fit_transform(X[train_i..." | |
| }, | |
| { | |
| "id": "leak_13", | |
| "description": "REINFORCE policy gradient loss function", | |
| "expected_leakage": true, | |
| "predicted_leakage": true, | |
| "status": "[PASS] TRUE POSITIVE", | |
| "sample_snippet": "```python\ndef compute_policy_loss(log_probs, returns):\n loss = []\n for lp, R in zip(log_probs, returns):\n l..." | |
| }, | |
| { | |
| "id": "leak_14", | |
| "description": "Value iteration multi-nested loop", | |
| "expected_leakage": true, | |
| "predicted_leakage": true, | |
| "status": "[PASS] TRUE POSITIVE", | |
| "sample_snippet": "```python\nwhile delta > theta:\n delta = 0\n for s in states:\n v = V[s]\n V[s] = max(sum(P * (R + gamma..." | |
| }, | |
| { | |
| "id": "leak_15", | |
| "description": "Hidden Markov Model forward variable recursion", | |
| "expected_leakage": true, | |
| "predicted_leakage": true, | |
| "status": "[PASS] TRUE POSITIVE", | |
| "sample_snippet": "```python\ndef forward_pass(obs, A, B, pi):\n alpha = np.zeros((len(obs), len(pi)))\n alpha[0] = pi * B[:, obs[0]]\n ..." | |
| }, | |
| { | |
| "id": "leak_16", | |
| "description": "K-Means cluster update step", | |
| "expected_leakage": true, | |
| "predicted_leakage": true, | |
| "status": "[PASS] TRUE POSITIVE", | |
| "sample_snippet": "```python\ndef update_centroids(X, labels, k):\n new_centroids = np.zeros((k, X.shape[1]))\n for i in range(k):\n ..." | |
| }, | |
| { | |
| "id": "leak_17", | |
| "description": "Uniform Cost Search priority queue fix", | |
| "expected_leakage": true, | |
| "predicted_leakage": true, | |
| "status": "[PASS] TRUE POSITIVE", | |
| "sample_snippet": "```python\ndef ucs(start, goal, graph):\n pq = [(0, start, [start])]\n visited = set()\n while pq:\n cost, no..." | |
| }, | |
| { | |
| "id": "leak_18", | |
| "description": "Naive Bayes log-likelihood scoring function", | |
| "expected_leakage": true, | |
| "predicted_leakage": true, | |
| "status": "[PASS] TRUE POSITIVE", | |
| "sample_snippet": "```python\ndef predict_log_proba(x, priors, conditionals):\n scores = np.log(priors.copy())\n for c in range(len(prio..." | |
| }, | |
| { | |
| "id": "leak_19", | |
| "description": "Backprop sigmoid gradient manual calculation", | |
| "expected_leakage": true, | |
| "predicted_leakage": true, | |
| "status": "[PASS] TRUE POSITIVE", | |
| "sample_snippet": "```python\ndef backward(self, X, y, a1, a2):\n m = X.shape[0]\n dz2 = a2 - y\n dW2 = (1 / m) * np.dot(dz2, a1.T)\n ..." | |
| }, | |
| { | |
| "id": "leak_20", | |
| "description": "Softmax temperature sampling implementation", | |
| "expected_leakage": true, | |
| "predicted_leakage": true, | |
| "status": "[PASS] TRUE POSITIVE", | |
| "sample_snippet": "```python\ndef sample_with_temperature(logits, temperature=0.7):\n scaled = logits / temperature\n probs = np.exp(sca..." | |
| }, | |
| { | |
| "id": "leak_21", | |
| "description": "Gini impurity computation function", | |
| "expected_leakage": true, | |
| "predicted_leakage": true, | |
| "status": "[PASS] TRUE POSITIVE", | |
| "sample_snippet": "```python\ndef gini(y):\n probs = [np.mean(y == c) for c in np.unique(y)]\n return 1.0 - sum(p**2 for p in probs)\n```" | |
| }, | |
| { | |
| "id": "leak_22", | |
| "description": "Q-table dictionary lookup and update helper", | |
| "expected_leakage": true, | |
| "predicted_leakage": true, | |
| "status": "[PASS] TRUE POSITIVE", | |
| "sample_snippet": "```python\ndef update_q(q_dict, s, a, r, s_prime, alpha, gamma):\n curr = q_dict.get((s, a), 0.0)\n next_max = max(q_..." | |
| }, | |
| { | |
| "id": "leak_23", | |
| "description": "Unparseable pseudocode snippet with code keywords and multi-line control flow", | |
| "expected_leakage": true, | |
| "predicted_leakage": true, | |
| "status": "[PASS] TRUE POSITIVE", | |
| "sample_snippet": "Here is the raw code snippet:\n```\ndef fix_bounds(grid, x, y):\n if x < 0 or x >= len(grid):\n return False\n i..." | |
| }, | |
| { | |
| "id": "leak_24", | |
| "description": "Tensor detachment autograd fix in custom trainer", | |
| "expected_leakage": true, | |
| "predicted_leakage": true, | |
| "status": "[PASS] TRUE POSITIVE", | |
| "sample_snippet": "```python\ndef step(self, batch):\n loss = self.model(batch)\n loss.backward()\n self.optimizer.step()\n return l..." | |
| }, | |
| { | |
| "id": "leak_25", | |
| "description": "Minimax with alpha-beta pruning complete wrapper", | |
| "expected_leakage": true, | |
| "predicted_leakage": true, | |
| "status": "[PASS] TRUE POSITIVE", | |
| "sample_snippet": "```python\ndef get_best_move(board, depth):\n best_val = -float('inf')\n best_move = None\n for move in board.legal..." | |
| }, | |
| { | |
| "id": "non_leak_01", | |
| "description": "Pure Socratic dialogue with questions", | |
| "expected_leakage": false, | |
| "predicted_leakage": false, | |
| "status": "[PASS] TRUE NEGATIVE", | |
| "sample_snippet": "What does your priority queue sort on? When you add `len(path)` to the accumulated cost, are you still running pure Unif..." | |
| }, | |
| { | |
| "id": "non_leak_02", | |
| "description": "Inline backticks referencing variable names only", | |
| "expected_leakage": false, | |
| "predicted_leakage": false, | |
| "status": "[PASS] TRUE NEGATIVE", | |
| "sample_snippet": "Check your cutoff condition: is it `alpha >= beta` or `alpha > beta`? Think about what happens when `alpha` equals `beta..." | |
| }, | |
| { | |
| "id": "non_leak_03", | |
| "description": "Mathematical formula using LaTeX notation", | |
| "expected_leakage": false, | |
| "predicted_leakage": false, | |
| "status": "[PASS] TRUE NEGATIVE", | |
| "sample_snippet": "Recall the Bellman equation: $Q(s, a) = r + \\gamma \\max_{a'} Q(s', a')$. Which term represents the immediate reward vers..." | |
| }, | |
| { | |
| "id": "non_leak_04", | |
| "description": "Single-line code fence containing only an equation / expression without functions or loops", | |
| "expected_leakage": false, | |
| "predicted_leakage": false, | |
| "status": "[PASS] TRUE NEGATIVE", | |
| "sample_snippet": "Consider the shape of your tensor before the linear layer:\n```\nExpected shape: (batch_size, num_features)\n```\nWhat is yo..." | |
| }, | |
| { | |
| "id": "non_leak_05", | |
| "description": "Guided debugging steps in bullet points", | |
| "expected_leakage": false, | |
| "predicted_leakage": false, | |
| "status": "[PASS] TRUE NEGATIVE", | |
| "sample_snippet": "Let's debug this step-by-step:\n1. Print the shape of `x` after the convolution.\n2. Calculate the spatial dimensions: $(W..." | |
| }, | |
| { | |
| "id": "non_leak_06", | |
| "description": "Conceptual explanation of vanishing gradients", | |
| "expected_leakage": false, | |
| "predicted_leakage": false, | |
| "status": "[PASS] TRUE NEGATIVE", | |
| "sample_snippet": "When you use the sigmoid activation with large initial weights, the pre-activation $z$ becomes very large. What is the d..." | |
| }, | |
| { | |
| "id": "non_leak_07", | |
| "description": "Explaining A* heuristic admissibility without code", | |
| "expected_leakage": false, | |
| "predicted_leakage": false, | |
| "status": "[PASS] TRUE NEGATIVE", | |
| "sample_snippet": "For A* to guarantee the optimal path, the heuristic $h(n)$ must be admissible ($h(n) \\le h^*(n)$). If your heuristic mul..." | |
| }, | |
| { | |
| "id": "non_leak_08", | |
| "description": "Socratic question about Viterbi transition index order", | |
| "expected_leakage": false, | |
| "predicted_leakage": false, | |
| "status": "[PASS] TRUE NEGATIVE", | |
| "sample_snippet": "Does $A[i, j]$ represent the transition probability from state $i$ to state $j$, or from $j$ to $i$? Check how your loop..." | |
| }, | |
| { | |
| "id": "non_leak_09", | |
| "description": "Guidance on train vs. eval mode in PyTorch", | |
| "expected_leakage": false, | |
| "predicted_leakage": false, | |
| "status": "[PASS] TRUE NEGATIVE", | |
| "sample_snippet": "Why do your test predictions change on every forward pass? Does your model contain stochastic layers like `nn.Dropout`? ..." | |
| }, | |
| { | |
| "id": "non_leak_10", | |
| "description": "Socratic prompt on decision tree weighted entropy", | |
| "expected_leakage": false, | |
| "predicted_leakage": false, | |
| "status": "[PASS] TRUE NEGATIVE", | |
| "sample_snippet": "If one child node contains 95 samples and the other child contains only 5 samples, should their impurities contribute eq..." | |
| }, | |
| { | |
| "id": "non_leak_11", | |
| "description": "Question on CrossEntropyLoss logits vs. softmax", | |
| "expected_leakage": false, | |
| "predicted_leakage": false, | |
| "status": "[PASS] TRUE NEGATIVE", | |
| "sample_snippet": "PyTorch's `nn.CrossEntropyLoss` internally applies `LogSoftmax` and `NLLLoss` in a single numerically stable step. What ..." | |
| }, | |
| { | |
| "id": "non_leak_12", | |
| "description": "Short code fence with 1-line mathematical formula", | |
| "expected_leakage": false, | |
| "predicted_leakage": false, | |
| "status": "[PASS] TRUE NEGATIVE", | |
| "sample_snippet": "Remember the update formula:\n```\nTD Target = R + gamma * max_a Q(S', a)\n```\nWhich term is multiplied by gamma?" | |
| }, | |
| { | |
| "id": "non_leak_13", | |
| "description": "Guided trace exercise for graph search", | |
| "expected_leakage": false, | |
| "predicted_leakage": false, | |
| "status": "[PASS] TRUE NEGATIVE", | |
| "sample_snippet": "Try tracing a simple 3-node cycle: $A \\to B \\to C \\to A$. If node $B$ is popped, when should its neighbors be added to t..." | |
| }, | |
| { | |
| "id": "non_leak_14", | |
| "description": "Explanation of data leakage in preprocessing", | |
| "expected_leakage": false, | |
| "predicted_leakage": false, | |
| "status": "[PASS] TRUE NEGATIVE", | |
| "sample_snippet": "When you fit `StandardScaler` on the whole dataset before splitting into train and test sets, what information from the ..." | |
| }, | |
| { | |
| "id": "non_leak_15", | |
| "description": "Socratic question on epsilon decay schedule", | |
| "expected_leakage": false, | |
| "predicted_leakage": false, | |
| "status": "[PASS] TRUE NEGATIVE", | |
| "sample_snippet": "If $\\epsilon = 1.0$ and you subtract 0.9 on the first episode, what is your exploration rate on episode 2? Did you inten..." | |
| }, | |
| { | |
| "id": "non_leak_16", | |
| "description": "Clarifying terminal state masking in RL", | |
| "expected_leakage": false, | |
| "predicted_leakage": false, | |
| "status": "[PASS] TRUE NEGATIVE", | |
| "sample_snippet": "When an episode terminates (`done = True`), is there any future state to transition to? What should the bootstrapped val..." | |
| }, | |
| { | |
| "id": "non_leak_17", | |
| "description": "Explaining zero-frequency problem in Naive Bayes", | |
| "expected_leakage": false, | |
| "predicted_leakage": false, | |
| "status": "[PASS] TRUE NEGATIVE", | |
| "sample_snippet": "If a word never appears in the training examples for class $C$, what is $P(w | C)$ without smoothing? What happens when ..." | |
| }, | |
| { | |
| "id": "non_leak_18", | |
| "description": "Code snippet showing only student's error message", | |
| "expected_leakage": false, | |
| "predicted_leakage": false, | |
| "status": "[PASS] TRUE NEGATIVE", | |
| "sample_snippet": "Notice the error you received:\n```\nRuntimeError: Expected size [12, 10] but got [1, 120]\n```\nWhy did the batch size of 1..." | |
| }, | |
| { | |
| "id": "non_leak_19", | |
| "description": "Discussion of L1 vs L2 regularization shrinkage", | |
| "expected_leakage": false, | |
| "predicted_leakage": false, | |
| "status": "[PASS] TRUE NEGATIVE", | |
| "sample_snippet": "How does L1 regularization differ from L2 regularization in terms of weight sparsity? Why does the derivative of $|w|$ p..." | |
| }, | |
| { | |
| "id": "non_leak_20", | |
| "description": "Socratic questioning on Bayesian network d-separation", | |
| "expected_leakage": false, | |
| "predicted_leakage": false, | |
| "status": "[PASS] TRUE NEGATIVE", | |
| "sample_snippet": "In the collider structure $A \\to C \\leftarrow B$, are $A$ and $B$ marginally independent? What happens to the active pat..." | |
| }, | |
| { | |
| "id": "non_leak_21", | |
| "description": "Guided inquiry on gradient accumulation resetting", | |
| "expected_leakage": false, | |
| "predicted_leakage": false, | |
| "status": "[PASS] TRUE NEGATIVE", | |
| "sample_snippet": "In PyTorch, `loss.backward()` accumulates gradients into `.grad` buffers rather than overwriting them. Where in your min..." | |
| }, | |
| { | |
| "id": "non_leak_22", | |
| "description": "Explaining difference between BFS and DFS queue structures", | |
| "expected_leakage": false, | |
| "predicted_leakage": false, | |
| "status": "[PASS] TRUE NEGATIVE", | |
| "sample_snippet": "BFS explores nodes level-by-level using a FIFO queue (`collections.deque`), whereas DFS uses a LIFO stack. Why does a FI..." | |
| }, | |
| { | |
| "id": "non_leak_23", | |
| "description": "Reflective question on loss reduction averaging", | |
| "expected_leakage": false, | |
| "predicted_leakage": false, | |
| "status": "[PASS] TRUE NEGATIVE", | |
| "sample_snippet": "If you use `reduction='sum'`, does the loss scale with the number of samples in the mini-batch? How does that affect you..." | |
| }, | |
| { | |
| "id": "non_leak_24", | |
| "description": "Minimax sign convention explanation", | |
| "expected_leakage": false, | |
| "predicted_leakage": false, | |
| "status": "[PASS] TRUE NEGATIVE", | |
| "sample_snippet": "If `state.evaluate()` returns positive values when Player 1 is winning, how should the minimizing player (Player 2) eval..." | |
| }, | |
| { | |
| "id": "non_leak_25", | |
| "description": "Prompting student to inspect learning rate magnitude", | |
| "expected_leakage": false, | |
| "predicted_leakage": false, | |
| "status": "[PASS] TRUE NEGATIVE", | |
| "sample_snippet": "If your network weights explode to `inf` within 3 iterations, check the scale of your learning rate. Try reducing $\\alph..." | |
| } | |
| ] | |
| } |