{
  "meta": {
    "model": "meta-llama/Llama-2-7b-chat-hf",
    "device": "cuda",
    "dtype": "fp32",
    "layer": 10,
    "layers_path": "model.layers",
    "seed": 123,
    "task": "humaneval",
    "eval_mode": "gen_code_compile",
    "eval_meta": {
      "hf_id": "openai_humaneval",
      "split": "test",
      "n_total": 164
    },
    "n_eval_loaded": 164,
    "n_scanned": 164,
    "base_acc_scan": 0.3475609756097561,
    "ablt_acc_scan": 0.31097560975609756,
    "flips_total": 38,
    "flips_used": 38,
    "patch_steps": [
      0,
      1,
      2,
      3
    ],
    "patch_n_steps": 4,
    "Qs_path": "Q_shared_layer10.npy",
    "Qs_shape": [
      4096,
      97
    ],
    "gold_text_prefix": " ",
    "dist_text_prefix": " ",
    "gold_max_tokens": 0,
    "distractor_mode": "next_gold",
    "answer_prefix_effective": "\nFinal answer:",
    "max_new_tokens_effective": 256,
    "run_coeff_controls": false,
    "use_benchmark_loader": false,
    "hf_id": "openai_humaneval",
    "hf_split": "test"
  },
  "summary_on_flips": {
    "patched_self": {
      "n": 38,
      "rescued": 8,
      "rescued_pct": 21.05263157894737
    },
    "control_time_shuffled": {
      "n": 38,
      "rescued": 8,
      "rescued_pct": 21.05263157894737
    },
    "control_shared_randvec": {
      "n": 38,
      "rescued": 10,
      "rescued_pct": 26.31578947368421
    },
    "control_rand_subspace": {
      "n": 38,
      "rescued": 7,
      "rescued_pct": 18.42105263157895
    },
    "control_patch_nonshared": {
      "n": 38,
      "rescued": 2,
      "rescued_pct": 5.2631578947368425
    }
  },
  "scan_rows": [
    {
      "ex_id": "openai_humaneval-test-18",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-31",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-158",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-43",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-39",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-15",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-151",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-101",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-34",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 232
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-52",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-113",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-119",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-83",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-116",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-56",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-131",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 191
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-1",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-159",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 163
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-123",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-23",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 227
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-124",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-38",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-84",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-41",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 174
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-134",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-2",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-80",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-74",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-162",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-138",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-87",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-145",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-54",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-109",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-102",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-62",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-129",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-110",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 208
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-4",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 222
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-8",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-97",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-137",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-96",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-65",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 237
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-114",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-117",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-155",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 233
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-37",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-115",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-86",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-85",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-53",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-130",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-139",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-81",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-19",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-69",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-122",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-108",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-48",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 244
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-17",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-95",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-91",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-79",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-14",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-77",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-3",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 156
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-146",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-57",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-143",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-0",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-106",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-50",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-58",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-147",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-160",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-67",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-55",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-118",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 204
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-154",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-25",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-36",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-63",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-132",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-21",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-9",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-72",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-128",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-126",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-70",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-40",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-66",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-71",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-107",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-32",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-11",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-47",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-22",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-140",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-30",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-10",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-112",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-111",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-28",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-135",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-94",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-13",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-7",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-157",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-49",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-120",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-89",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-98",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-100",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-64",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-103",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-125",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-51",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 229
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-90",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-152",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-24",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-20",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-148",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-92",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-26",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 235
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-16",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-142",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-99",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-127",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 171
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-156",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-141",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-78",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-68",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-93",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-60",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-82",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-59",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-149",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 205
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-42",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-163",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-133",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-161",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-29",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 239
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-27",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-61",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 253
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-104",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-45",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 213
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-75",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-6",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-136",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-33",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-44",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-88",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 233
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-12",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 177
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-105",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-121",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-144",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-73",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-76",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-5",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": true,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-46",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-150",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-35",
      "baseline": {
        "compile_ok": false,
        "n_gen_tokens": 256
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    },
    {
      "ex_id": "openai_humaneval-test-153",
      "baseline": {
        "compile_ok": true,
        "n_gen_tokens": 242
      },
      "ablated": {
        "compile_ok": false,
        "n_gen_tokens": 256
      }
    }
  ],
  "flip_rows": [
    {
      "ex_id": "openai_humaneval-test-31",
      "baseline": {
        "compile_ok": true
      },
      "ablated": {
        "compile_ok": false
      },
      "patched_self": {
        "compile_ok": false
      },
      "control_time_shuffled": {
        "compile_ok": false
      },
      "control_shared_randvec": {
        "compile_ok": false
      },
      "control_rand_subspace": {
        "compile_ok": false
      },
      "control_patch_nonshared": {
        "compile_ok": false
      }
    },
    {
      "ex_id": "openai_humaneval-test-158",
      "baseline": {
        "compile_ok": true
      },
      "ablated": {
        "compile_ok": false
      },
      "patched_self": {
        "compile_ok": false
      },
      "control_time_shuffled": {
        "compile_ok": false
      },
      "control_shared_randvec": {
        "compile_ok": false
      },
      "control_rand_subspace": {
        "compile_ok": false
      },
      "control_patch_nonshared": {
        "compile_ok": false
      }
    },
    {
      "ex_id": "openai_humaneval-test-39",
      "baseline": {
        "compile_ok": true
      },
      "ablated": {
        "compile_ok": false
      },
      "patched_self": {
        "compile_ok": false
      },
      "control_time_shuffled": {
        "compile_ok": false
      },
      "control_shared_randvec": {
        "compile_ok": false
      },
      "control_rand_subspace": {
        "compile_ok": false
      },
      "control_patch_nonshared": {
        "compile_ok": false
      }
    },
    {
      "ex_id": "openai_humaneval-test-101",
      "baseline": {
        "compile_ok": true
      },
      "ablated": {
        "compile_ok": false
      },
      "patched_self": {
        "compile_ok": false
      },
      "control_time_shuffled": {
        "compile_ok": false
      },
      "control_shared_randvec": {
        "compile_ok": true
      },
      "control_rand_subspace": {
        "compile_ok": false
      },
      "control_patch_nonshared": {
        "compile_ok": false
      }
    },
    {
      "ex_id": "openai_humaneval-test-56",
      "baseline": {
        "compile_ok": true
      },
      "ablated": {
        "compile_ok": false
      },
      "patched_self": {
        "compile_ok": false
      },
      "control_time_shuffled": {
        "compile_ok": false
      },
      "control_shared_randvec": {
        "compile_ok": false
      },
      "control_rand_subspace": {
        "compile_ok": false
      },
      "control_patch_nonshared": {
        "compile_ok": false
      }
    },
    {
      "ex_id": "openai_humaneval-test-1",
      "baseline": {
        "compile_ok": true
      },
      "ablated": {
        "compile_ok": false
      },
      "patched_self": {
        "compile_ok": true
      },
      "control_time_shuffled": {
        "compile_ok": false
      },
      "control_shared_randvec": {
        "compile_ok": true
      },
      "control_rand_subspace": {
        "compile_ok": true
      },
      "control_patch_nonshared": {
        "compile_ok": false
      }
    },
    {
      "ex_id": "openai_humaneval-test-159",
      "baseline": {
        "compile_ok": true
      },
      "ablated": {
        "compile_ok": false
      },
      "patched_self": {
        "compile_ok": false
      },
      "control_time_shuffled": {
        "compile_ok": true
      },
      "control_shared_randvec": {
        "compile_ok": false
      },
      "control_rand_subspace": {
        "compile_ok": false
      },
      "control_patch_nonshared": {
        "compile_ok": false
      }
    },
    {
      "ex_id": "openai_humaneval-test-123",
      "baseline": {
        "compile_ok": true
      },
      "ablated": {
        "compile_ok": false
      },
      "patched_self": {
        "compile_ok": false
      },
      "control_time_shuffled": {
        "compile_ok": false
      },
      "control_shared_randvec": {
        "compile_ok": true
      },
      "control_rand_subspace": {
        "compile_ok": false
      },
      "control_patch_nonshared": {
        "compile_ok": false
      }
    },
    {
      "ex_id": "openai_humaneval-test-38",
      "baseline": {
        "compile_ok": true
      },
      "ablated": {
        "compile_ok": false
      },
      "patched_self": {
        "compile_ok": false
      },
      "control_time_shuffled": {
        "compile_ok": false
      },
      "control_shared_randvec": {
        "compile_ok": true
      },
      "control_rand_subspace": {
        "compile_ok": false
      },
      "control_patch_nonshared": {
        "compile_ok": false
      }
    },
    {
      "ex_id": "openai_humaneval-test-84",
      "baseline": {
        "compile_ok": true
      },
      "ablated": {
        "compile_ok": false
      },
      "patched_self": {
        "compile_ok": true
      },
      "control_time_shuffled": {
        "compile_ok": false
      },
      "control_shared_randvec": {
        "compile_ok": false
      },
      "control_rand_subspace": {
        "compile_ok": true
      },
      "control_patch_nonshared": {
        "compile_ok": false
      }
    },
    {
      "ex_id": "openai_humaneval-test-134",
      "baseline": {
        "compile_ok": true
      },
      "ablated": {
        "compile_ok": false
      },
      "patched_self": {
        "compile_ok": true
      },
      "control_time_shuffled": {
        "compile_ok": true
      },
      "control_shared_randvec": {
        "compile_ok": false
      },
      "control_rand_subspace": {
        "compile_ok": false
      },
      "control_patch_nonshared": {
        "compile_ok": true
      }
    },
    {
      "ex_id": "openai_humaneval-test-54",
      "baseline": {
        "compile_ok": true
      },
      "ablated": {
        "compile_ok": false
      },
      "patched_self": {
        "compile_ok": false
      },
      "control_time_shuffled": {
        "compile_ok": false
      },
      "control_shared_randvec": {
        "compile_ok": false
      },
      "control_rand_subspace": {
        "compile_ok": false
      },
      "control_patch_nonshared": {
        "compile_ok": false
      }
    },
    {
      "ex_id": "openai_humaneval-test-110",
      "baseline": {
        "compile_ok": true
      },
      "ablated": {
        "compile_ok": false
      },
      "patched_self": {
        "compile_ok": false
      },
      "control_time_shuffled": {
        "compile_ok": false
      },
      "control_shared_randvec": {
        "compile_ok": false
      },
      "control_rand_subspace": {
        "compile_ok": false
      },
      "control_patch_nonshared": {
        "compile_ok": false
      }
    },
    {
      "ex_id": "openai_humaneval-test-86",
      "baseline": {
        "compile_ok": true
      },
      "ablated": {
        "compile_ok": false
      },
      "patched_self": {
        "compile_ok": false
      },
      "control_time_shuffled": {
        "compile_ok": false
      },
      "control_shared_randvec": {
        "compile_ok": true
      },
      "control_rand_subspace": {
        "compile_ok": true
      },
      "control_patch_nonshared": {
        "compile_ok": true
      }
    },
    {
      "ex_id": "openai_humaneval-test-53",
      "baseline": {
        "compile_ok": true
      },
      "ablated": {
        "compile_ok": false
      },
      "patched_self": {
        "compile_ok": false
      },
      "control_time_shuffled": {
        "compile_ok": false
      },
      "control_shared_randvec": {
        "compile_ok": false
      },
      "control_rand_subspace": {
        "compile_ok": false
      },
      "control_patch_nonshared": {
        "compile_ok": false
      }
    },
    {
      "ex_id": "openai_humaneval-test-81",
      "baseline": {
        "compile_ok": true
      },
      "ablated": {
        "compile_ok": false
      },
      "patched_self": {
        "compile_ok": false
      },
      "control_time_shuffled": {
        "compile_ok": false
      },
      "control_shared_randvec": {
        "compile_ok": false
      },
      "control_rand_subspace": {
        "compile_ok": false
      },
      "control_patch_nonshared": {
        "compile_ok": false
      }
    },
    {
      "ex_id": "openai_humaneval-test-122",
      "baseline": {
        "compile_ok": true
      },
      "ablated": {
        "compile_ok": false
      },
      "patched_self": {
        "compile_ok": false
      },
      "control_time_shuffled": {
        "compile_ok": false
      },
      "control_shared_randvec": {
        "compile_ok": false
      },
      "control_rand_subspace": {
        "compile_ok": false
      },
      "control_patch_nonshared": {
        "compile_ok": false
      }
    },
    {
      "ex_id": "openai_humaneval-test-17",
      "baseline": {
        "compile_ok": true
      },
      "ablated": {
        "compile_ok": false
      },
      "patched_self": {
        "compile_ok": true
      },
      "control_time_shuffled": {
        "compile_ok": false
      },
      "control_shared_randvec": {
        "compile_ok": false
      },
      "control_rand_subspace": {
        "compile_ok": false
      },
      "control_patch_nonshared": {
        "compile_ok": false
      }
    },
    {
      "ex_id": "openai_humaneval-test-95",
      "baseline": {
        "compile_ok": true
      },
      "ablated": {
        "compile_ok": false
      },
      "patched_self": {
        "compile_ok": false
      },
      "control_time_shuffled": {
        "compile_ok": false
      },
      "control_shared_randvec": {
        "compile_ok": true
      },
      "control_rand_subspace": {
        "compile_ok": false
      },
      "control_patch_nonshared": {
        "compile_ok": false
      }
    },
    {
      "ex_id": "openai_humaneval-test-3",
      "baseline": {
        "compile_ok": true
      },
      "ablated": {
        "compile_ok": false
      },
      "patched_self": {
        "compile_ok": false
      },
      "control_time_shuffled": {
        "compile_ok": true
      },
      "control_shared_randvec": {
        "compile_ok": false
      },
      "control_rand_subspace": {
        "compile_ok": false
      },
      "control_patch_nonshared": {
        "compile_ok": false
      }
    },
    {
      "ex_id": "openai_humaneval-test-58",
      "baseline": {
        "compile_ok": true
      },
      "ablated": {
        "compile_ok": false
      },
      "patched_self": {
        "compile_ok": false
      },
      "control_time_shuffled": {
        "compile_ok": true
      },
      "control_shared_randvec": {
        "compile_ok": true
      },
      "control_rand_subspace": {
        "compile_ok": false
      },
      "control_patch_nonshared": {
        "compile_ok": false
      }
    },
    {
      "ex_id": "openai_humaneval-test-147",
      "baseline": {
        "compile_ok": true
      },
      "ablated": {
        "compile_ok": false
      },
      "patched_self": {
        "compile_ok": true
      },
      "control_time_shuffled": {
        "compile_ok": false
      },
      "control_shared_randvec": {
        "compile_ok": false
      },
      "control_rand_subspace": {
        "compile_ok": false
      },
      "control_patch_nonshared": {
        "compile_ok": false
      }
    },
    {
      "ex_id": "openai_humaneval-test-55",
      "baseline": {
        "compile_ok": true
      },
      "ablated": {
        "compile_ok": false
      },
      "patched_self": {
        "compile_ok": false
      },
      "control_time_shuffled": {
        "compile_ok": false
      },
      "control_shared_randvec": {
        "compile_ok": false
      },
      "control_rand_subspace": {
        "compile_ok": false
      },
      "control_patch_nonshared": {
        "compile_ok": false
      }
    },
    {
      "ex_id": "openai_humaneval-test-118",
      "baseline": {
        "compile_ok": true
      },
      "ablated": {
        "compile_ok": false
      },
      "patched_self": {
        "compile_ok": false
      },
      "control_time_shuffled": {
        "compile_ok": false
      },
      "control_shared_randvec": {
        "compile_ok": false
      },
      "control_rand_subspace": {
        "compile_ok": false
      },
      "control_patch_nonshared": {
        "compile_ok": false
      }
    },
    {
      "ex_id": "openai_humaneval-test-30",
      "baseline": {
        "compile_ok": true
      },
      "ablated": {
        "compile_ok": false
      },
      "patched_self": {
        "compile_ok": false
      },
      "control_time_shuffled": {
        "compile_ok": false
      },
      "control_shared_randvec": {
        "compile_ok": false
      },
      "control_rand_subspace": {
        "compile_ok": false
      },
      "control_patch_nonshared": {
        "compile_ok": false
      }
    },
    {
      "ex_id": "openai_humaneval-test-112",
      "baseline": {
        "compile_ok": true
      },
      "ablated": {
        "compile_ok": false
      },
      "patched_self": {
        "compile_ok": true
      },
      "control_time_shuffled": {
        "compile_ok": false
      },
      "control_shared_randvec": {
        "compile_ok": true
      },
      "control_rand_subspace": {
        "compile_ok": false
      },
      "control_patch_nonshared": {
        "compile_ok": false
      }
    },
    {
      "ex_id": "openai_humaneval-test-49",
      "baseline": {
        "compile_ok": true
      },
      "ablated": {
        "compile_ok": false
      },
      "patched_self": {
        "compile_ok": false
      },
      "control_time_shuffled": {
        "compile_ok": false
      },
      "control_shared_randvec": {
        "compile_ok": false
      },
      "control_rand_subspace": {
        "compile_ok": false
      },
      "control_patch_nonshared": {
        "compile_ok": false
      }
    },
    {
      "ex_id": "openai_humaneval-test-64",
      "baseline": {
        "compile_ok": true
      },
      "ablated": {
        "compile_ok": false
      },
      "patched_self": {
        "compile_ok": false
      },
      "control_time_shuffled": {
        "compile_ok": false
      },
      "control_shared_randvec": {
        "compile_ok": false
      },
      "control_rand_subspace": {
        "compile_ok": false
      },
      "control_patch_nonshared": {
        "compile_ok": false
      }
    },
    {
      "ex_id": "openai_humaneval-test-92",
      "baseline": {
        "compile_ok": true
      },
      "ablated": {
        "compile_ok": false
      },
      "patched_self": {
        "compile_ok": false
      },
      "control_time_shuffled": {
        "compile_ok": true
      },
      "control_shared_randvec": {
        "compile_ok": false
      },
      "control_rand_subspace": {
        "compile_ok": true
      },
      "control_patch_nonshared": {
        "compile_ok": false
      }
    },
    {
      "ex_id": "openai_humaneval-test-127",
      "baseline": {
        "compile_ok": true
      },
      "ablated": {
        "compile_ok": false
      },
      "patched_self": {
        "compile_ok": false
      },
      "control_time_shuffled": {
        "compile_ok": true
      },
      "control_shared_randvec": {
        "compile_ok": false
      },
      "control_rand_subspace": {
        "compile_ok": false
      },
      "control_patch_nonshared": {
        "compile_ok": false
      }
    },
    {
      "ex_id": "openai_humaneval-test-68",
      "baseline": {
        "compile_ok": true
      },
      "ablated": {
        "compile_ok": false
      },
      "patched_self": {
        "compile_ok": false
      },
      "control_time_shuffled": {
        "compile_ok": false
      },
      "control_shared_randvec": {
        "compile_ok": false
      },
      "control_rand_subspace": {
        "compile_ok": false
      },
      "control_patch_nonshared": {
        "compile_ok": false
      }
    },
    {
      "ex_id": "openai_humaneval-test-93",
      "baseline": {
        "compile_ok": true
      },
      "ablated": {
        "compile_ok": false
      },
      "patched_self": {
        "compile_ok": false
      },
      "control_time_shuffled": {
        "compile_ok": false
      },
      "control_shared_randvec": {
        "compile_ok": false
      },
      "control_rand_subspace": {
        "compile_ok": false
      },
      "control_patch_nonshared": {
        "compile_ok": false
      }
    },
    {
      "ex_id": "openai_humaneval-test-133",
      "baseline": {
        "compile_ok": true
      },
      "ablated": {
        "compile_ok": false
      },
      "patched_self": {
        "compile_ok": false
      },
      "control_time_shuffled": {
        "compile_ok": false
      },
      "control_shared_randvec": {
        "compile_ok": false
      },
      "control_rand_subspace": {
        "compile_ok": true
      },
      "control_patch_nonshared": {
        "compile_ok": false
      }
    },
    {
      "ex_id": "openai_humaneval-test-61",
      "baseline": {
        "compile_ok": true
      },
      "ablated": {
        "compile_ok": false
      },
      "patched_self": {
        "compile_ok": false
      },
      "control_time_shuffled": {
        "compile_ok": false
      },
      "control_shared_randvec": {
        "compile_ok": true
      },
      "control_rand_subspace": {
        "compile_ok": false
      },
      "control_patch_nonshared": {
        "compile_ok": false
      }
    },
    {
      "ex_id": "openai_humaneval-test-104",
      "baseline": {
        "compile_ok": true
      },
      "ablated": {
        "compile_ok": false
      },
      "patched_self": {
        "compile_ok": true
      },
      "control_time_shuffled": {
        "compile_ok": true
      },
      "control_shared_randvec": {
        "compile_ok": false
      },
      "control_rand_subspace": {
        "compile_ok": false
      },
      "control_patch_nonshared": {
        "compile_ok": false
      }
    },
    {
      "ex_id": "openai_humaneval-test-12",
      "baseline": {
        "compile_ok": true
      },
      "ablated": {
        "compile_ok": false
      },
      "patched_self": {
        "compile_ok": false
      },
      "control_time_shuffled": {
        "compile_ok": false
      },
      "control_shared_randvec": {
        "compile_ok": false
      },
      "control_rand_subspace": {
        "compile_ok": true
      },
      "control_patch_nonshared": {
        "compile_ok": false
      }
    },
    {
      "ex_id": "openai_humaneval-test-105",
      "baseline": {
        "compile_ok": true
      },
      "ablated": {
        "compile_ok": false
      },
      "patched_self": {
        "compile_ok": false
      },
      "control_time_shuffled": {
        "compile_ok": true
      },
      "control_shared_randvec": {
        "compile_ok": true
      },
      "control_rand_subspace": {
        "compile_ok": true
      },
      "control_patch_nonshared": {
        "compile_ok": false
      }
    },
    {
      "ex_id": "openai_humaneval-test-153",
      "baseline": {
        "compile_ok": true
      },
      "ablated": {
        "compile_ok": false
      },
      "patched_self": {
        "compile_ok": true
      },
      "control_time_shuffled": {
        "compile_ok": false
      },
      "control_shared_randvec": {
        "compile_ok": false
      },
      "control_rand_subspace": {
        "compile_ok": false
      },
      "control_patch_nonshared": {
        "compile_ok": false
      }
    }
  ]
}