artifacts/metrics/omac_4b_code2trivia__factorial_summary.json

{
  "aurc_by_seed": [
    {
      "aurc": 0.3987,
      "condition": "FROZEN_EVOLVED",
      "mean_post_shift_accuracy": 0.402,
      "n_post_shift": 500,
      "seed": 0
    },
    {
      "aurc": 0.375,
      "condition": "FROZEN_EVOLVED",
      "mean_post_shift_accuracy": 0.384,
      "n_post_shift": 500,
      "seed": 1
    },
    {
      "aurc": 0.3998,
      "condition": "FROZEN_EVOLVED",
      "mean_post_shift_accuracy": 0.412,
      "n_post_shift": 500,
      "seed": 2
    },
    {
      "aurc": 0.4085,
      "condition": "FROZEN_EVOLVED",
      "mean_post_shift_accuracy": 0.406,
      "n_post_shift": 500,
      "seed": 3
    },
    {
      "aurc": 0.3909,
      "condition": "FROZEN_EVOLVED",
      "mean_post_shift_accuracy": 0.39,
      "n_post_shift": 500,
      "seed": 4
    },
    {
      "aurc": 0.3628,
      "condition": "FROZEN_EVOLVED",
      "mean_post_shift_accuracy": 0.404,
      "n_post_shift": 500,
      "seed": 5
    },
    {
      "aurc": 0.4171,
      "condition": "FROZEN_EVOLVED",
      "mean_post_shift_accuracy": 0.408,
      "n_post_shift": 500,
      "seed": 6
    },
    {
      "aurc": 0.412,
      "condition": "FROZEN_EVOLVED",
      "mean_post_shift_accuracy": 0.408,
      "n_post_shift": 500,
      "seed": 7
    },
    {
      "aurc": 0.358,
      "condition": "FROZEN_EVOLVED",
      "mean_post_shift_accuracy": 0.394,
      "n_post_shift": 500,
      "seed": 8
    },
    {
      "aurc": 0.3819,
      "condition": "FROZEN_EVOLVED",
      "mean_post_shift_accuracy": 0.386,
      "n_post_shift": 500,
      "seed": 9
    },
    {
      "aurc": 0.4134,
      "condition": "NO_MEMORY",
      "mean_post_shift_accuracy": 0.414,
      "n_post_shift": 500,
      "seed": 0
    },
    {
      "aurc": 0.3994,
      "condition": "NO_MEMORY",
      "mean_post_shift_accuracy": 0.412,
      "n_post_shift": 500,
      "seed": 1
    },
    {
      "aurc": 0.3994,
      "condition": "NO_MEMORY",
      "mean_post_shift_accuracy": 0.414,
      "n_post_shift": 500,
      "seed": 2
    },
    {
      "aurc": 0.4265,
      "condition": "NO_MEMORY",
      "mean_post_shift_accuracy": 0.41,
      "n_post_shift": 500,
      "seed": 3
    },
    {
      "aurc": 0.4064,
      "condition": "NO_MEMORY",
      "mean_post_shift_accuracy": 0.414,
      "n_post_shift": 500,
      "seed": 4
    },
    {
      "aurc": 0.3771,
      "condition": "NO_MEMORY",
      "mean_post_shift_accuracy": 0.414,
      "n_post_shift": 500,
      "seed": 5
    },
    {
      "aurc": 0.4333,
      "condition": "NO_MEMORY",
      "mean_post_shift_accuracy": 0.414,
      "n_post_shift": 500,
      "seed": 6
    },
    {
      "aurc": 0.4188,
      "condition": "NO_MEMORY",
      "mean_post_shift_accuracy": 0.414,
      "n_post_shift": 500,
      "seed": 7
    },
    {
      "aurc": 0.3807,
      "condition": "NO_MEMORY",
      "mean_post_shift_accuracy": 0.412,
      "n_post_shift": 500,
      "seed": 8
    },
    {
      "aurc": 0.4199,
      "condition": "NO_MEMORY",
      "mean_post_shift_accuracy": 0.414,
      "n_post_shift": 500,
      "seed": 9
    },
    {
      "aurc": 0.4165,
      "condition": "ONLINE_JOINT",
      "mean_post_shift_accuracy": 0.41,
      "n_post_shift": 500,
      "seed": 0
    },
    {
      "aurc": 0.3797,
      "condition": "ONLINE_JOINT",
      "mean_post_shift_accuracy": 0.386,
      "n_post_shift": 500,
      "seed": 1
    },
    {
      "aurc": 0.3925,
      "condition": "ONLINE_JOINT",
      "mean_post_shift_accuracy": 0.404,
      "n_post_shift": 500,
      "seed": 2
    },
    {
      "aurc": 0.4146,
      "condition": "ONLINE_JOINT",
      "mean_post_shift_accuracy": 0.4,
      "n_post_shift": 500,
      "seed": 3
    },
    {
      "aurc": 0.4066,
      "condition": "ONLINE_JOINT",
      "mean_post_shift_accuracy": 0.396,
      "n_post_shift": 500,
      "seed": 4
    },
    {
      "aurc": 0.3495,
      "condition": "ONLINE_JOINT",
      "mean_post_shift_accuracy": 0.406,
      "n_post_shift": 500,
      "seed": 5
    },
    {
      "aurc": 0.435,
      "condition": "ONLINE_JOINT",
      "mean_post_shift_accuracy": 0.404,
      "n_post_shift": 500,
      "seed": 6
    },
    {
      "aurc": 0.4056,
      "condition": "ONLINE_JOINT",
      "mean_post_shift_accuracy": 0.402,
      "n_post_shift": 500,
      "seed": 7
    },
    {
      "aurc": 0.3616,
      "condition": "ONLINE_JOINT",
      "mean_post_shift_accuracy": 0.402,
      "n_post_shift": 500,
      "seed": 8
    },
    {
      "aurc": 0.3871,
      "condition": "ONLINE_JOINT",
      "mean_post_shift_accuracy": 0.396,
      "n_post_shift": 500,
      "seed": 9
    },
    {
      "aurc": 0.4034,
      "condition": "ONLINE_MANAGE_ONLY",
      "mean_post_shift_accuracy": 0.406,
      "n_post_shift": 500,
      "seed": 0
    },
    {
      "aurc": 0.3781,
      "condition": "ONLINE_MANAGE_ONLY",
      "mean_post_shift_accuracy": 0.388,
      "n_post_shift": 500,
      "seed": 1
    },
    {
      "aurc": 0.3939,
      "condition": "ONLINE_MANAGE_ONLY",
      "mean_post_shift_accuracy": 0.4,
      "n_post_shift": 500,
      "seed": 2
    },
    {
      "aurc": 0.4178,
      "condition": "ONLINE_MANAGE_ONLY",
      "mean_post_shift_accuracy": 0.406,
      "n_post_shift": 500,
      "seed": 3
    },
    {
      "aurc": 0.3951,
      "condition": "ONLINE_MANAGE_ONLY",
      "mean_post_shift_accuracy": 0.398,
      "n_post_shift": 500,
      "seed": 4
    },
    {
      "aurc": 0.3649,
      "condition": "ONLINE_MANAGE_ONLY",
      "mean_post_shift_accuracy": 0.404,
      "n_post_shift": 500,
      "seed": 5
    },
    {
      "aurc": 0.4389,
      "condition": "ONLINE_MANAGE_ONLY",
      "mean_post_shift_accuracy": 0.414,
      "n_post_shift": 500,
      "seed": 6
    },
    {
      "aurc": 0.404,
      "condition": "ONLINE_MANAGE_ONLY",
      "mean_post_shift_accuracy": 0.408,
      "n_post_shift": 500,
      "seed": 7
    },
    {
      "aurc": 0.3558,
      "condition": "ONLINE_MANAGE_ONLY",
      "mean_post_shift_accuracy": 0.394,
      "n_post_shift": 500,
      "seed": 8
    },
    {
      "aurc": 0.3842,
      "condition": "ONLINE_MANAGE_ONLY",
      "mean_post_shift_accuracy": 0.396,
      "n_post_shift": 500,
      "seed": 9
    }
  ],
  "conditions": {
    "FROZEN_EVOLVED": {
      "ci95_high": 0.4025,
      "ci95_low": 0.3789,
      "mean_aurc": 0.3905,
      "mean_exploration_rate": 0.0,
      "mean_ttr": 5.5,
      "n_seeds": 10,
      "recovery_rate": 1.0
    },
    "NO_MEMORY": {
      "ci95_high": 0.4178,
      "ci95_low": 0.3957,
      "mean_aurc": 0.4075,
      "mean_exploration_rate": 0.0,
      "mean_ttr": 6.7,
      "n_seeds": 10,
      "recovery_rate": 1.0
    },
    "ONLINE_JOINT": {
      "ci95_high": 0.409,
      "ci95_low": 0.3781,
      "mean_aurc": 0.3949,
      "mean_exploration_rate": 1.0,
      "mean_ttr": 5.6,
      "n_seeds": 10,
      "recovery_rate": 1.0
    },
    "ONLINE_MANAGE_ONLY": {
      "ci95_high": 0.4091,
      "ci95_low": 0.3789,
      "mean_aurc": 0.3936,
      "mean_exploration_rate": 0.9417,
      "mean_ttr": 5.5,
      "n_seeds": 10,
      "recovery_rate": 1.0
    }
  },
  "n_blocks_total": 3920,
  "n_episodes_total": 39160,
  "shift_episode": 479,
  "ttr_by_seed": [
    {
      "condition": "FROZEN_EVOLVED",
      "recovered": true,
      "seed": 0,
      "target_accuracy": 0.1729,
      "time_to_recovery": 5
    },
    {
      "condition": "FROZEN_EVOLVED",
      "recovered": true,
      "seed": 1,
      "target_accuracy": 0.186,
      "time_to_recovery": 9
    },
    {
      "condition": "FROZEN_EVOLVED",
      "recovered": true,
      "seed": 2,
      "target_accuracy": 0.1785,
      "time_to_recovery": 5
    },
    {
      "condition": "FROZEN_EVOLVED",
      "recovered": true,
      "seed": 3,
      "target_accuracy": 0.1353,
      "time_to_recovery": 5
    },
    {
      "condition": "FROZEN_EVOLVED",
      "recovered": true,
      "seed": 4,
      "target_accuracy": 0.1766,
      "time_to_recovery": 5
    },
    {
      "condition": "FROZEN_EVOLVED",
      "recovered": true,
      "seed": 5,
      "target_accuracy": 0.1898,
      "time_to_recovery": 6
    },
    {
      "condition": "FROZEN_EVOLVED",
      "recovered": true,
      "seed": 6,
      "target_accuracy": 0.1672,
      "time_to_recovery": 5
    },
    {
      "condition": "FROZEN_EVOLVED",
      "recovered": true,
      "seed": 7,
      "target_accuracy": 0.171,
      "time_to_recovery": 5
    },
    {
      "condition": "FROZEN_EVOLVED",
      "recovered": true,
      "seed": 8,
      "target_accuracy": 0.1672,
      "time_to_recovery": 5
    },
    {
      "condition": "FROZEN_EVOLVED",
      "recovered": true,
      "seed": 9,
      "target_accuracy": 0.1804,
      "time_to_recovery": 5
    },
    {
      "condition": "NO_MEMORY",
      "recovered": true,
      "seed": 0,
      "target_accuracy": 0.2161,
      "time_to_recovery": 5
    },
    {
      "condition": "NO_MEMORY",
      "recovered": true,
      "seed": 1,
      "target_accuracy": 0.2161,
      "time_to_recovery": 10
    },
    {
      "condition": "NO_MEMORY",
      "recovered": true,
      "seed": 2,
      "target_accuracy": 0.2161,
      "time_to_recovery": 5
    },
    {
      "condition": "NO_MEMORY",
      "recovered": true,
      "seed": 3,
      "target_accuracy": 0.2161,
      "time_to_recovery": 5
    },
    {
      "condition": "NO_MEMORY",
      "recovered": true,
      "seed": 4,
      "target_accuracy": 0.2161,
      "time_to_recovery": 5
    },
    {
      "condition": "NO_MEMORY",
      "recovered": true,
      "seed": 5,
      "target_accuracy": 0.2161,
      "time_to_recovery": 8
    },
    {
      "condition": "NO_MEMORY",
      "recovered": true,
      "seed": 6,
      "target_accuracy": 0.2142,
      "time_to_recovery": 5
    },
    {
      "condition": "NO_MEMORY",
      "recovered": true,
      "seed": 7,
      "target_accuracy": 0.2161,
      "time_to_recovery": 5
    },
    {
      "condition": "NO_MEMORY",
      "recovered": true,
      "seed": 8,
      "target_accuracy": 0.2161,
      "time_to_recovery": 10
    },
    {
      "condition": "NO_MEMORY",
      "recovered": true,
      "seed": 9,
      "target_accuracy": 0.2161,
      "time_to_recovery": 9
    },
    {
      "condition": "ONLINE_JOINT",
      "recovered": true,
      "seed": 0,
      "target_accuracy": 0.1804,
      "time_to_recovery": 5
    },
    {
      "condition": "ONLINE_JOINT",
      "recovered": true,
      "seed": 1,
      "target_accuracy": 0.1766,
      "time_to_recovery": 5
    },
    {
      "condition": "ONLINE_JOINT",
      "recovered": true,
      "seed": 2,
      "target_accuracy": 0.1747,
      "time_to_recovery": 5
    },
    {
      "condition": "ONLINE_JOINT",
      "recovered": true,
      "seed": 3,
      "target_accuracy": 0.1597,
      "time_to_recovery": 5
    },
    {
      "condition": "ONLINE_JOINT",
      "recovered": true,
      "seed": 4,
      "target_accuracy": 0.186,
      "time_to_recovery": 5
    },
    {
      "condition": "ONLINE_JOINT",
      "recovered": true,
      "seed": 5,
      "target_accuracy": 0.1597,
      "time_to_recovery": 6
    },
    {
      "condition": "ONLINE_JOINT",
      "recovered": true,
      "seed": 6,
      "target_accuracy": 0.1578,
      "time_to_recovery": 5
    },
    {
      "condition": "ONLINE_JOINT",
      "recovered": true,
      "seed": 7,
      "target_accuracy": 0.1672,
      "time_to_recovery": 5
    },
    {
      "condition": "ONLINE_JOINT",
      "recovered": true,
      "seed": 8,
      "target_accuracy": 0.1691,
      "time_to_recovery": 10
    },
    {
      "condition": "ONLINE_JOINT",
      "recovered": true,
      "seed": 9,
      "target_accuracy": 0.1785,
      "time_to_recovery": 5
    },
    {
      "condition": "ONLINE_MANAGE_ONLY",
      "recovered": true,
      "seed": 0,
      "target_accuracy": 0.1691,
      "time_to_recovery": 5
    },
    {
      "condition": "ONLINE_MANAGE_ONLY",
      "recovered": true,
      "seed": 1,
      "target_accuracy": 0.186,
      "time_to_recovery": 9
    },
    {
      "condition": "ONLINE_MANAGE_ONLY",
      "recovered": true,
      "seed": 2,
      "target_accuracy": 0.1785,
      "time_to_recovery": 5
    },
    {
      "condition": "ONLINE_MANAGE_ONLY",
      "recovered": true,
      "seed": 3,
      "target_accuracy": 0.1635,
      "time_to_recovery": 5
    },
    {
      "condition": "ONLINE_MANAGE_ONLY",
      "recovered": true,
      "seed": 4,
      "target_accuracy": 0.1804,
      "time_to_recovery": 5
    },
    {
      "condition": "ONLINE_MANAGE_ONLY",
      "recovered": true,
      "seed": 5,
      "target_accuracy": 0.1935,
      "time_to_recovery": 6
    },
    {
      "condition": "ONLINE_MANAGE_ONLY",
      "recovered": true,
      "seed": 6,
      "target_accuracy": 0.1804,
      "time_to_recovery": 5
    },
    {
      "condition": "ONLINE_MANAGE_ONLY",
      "recovered": true,
      "seed": 7,
      "target_accuracy": 0.1747,
      "time_to_recovery": 5
    },
    {
      "condition": "ONLINE_MANAGE_ONLY",
      "recovered": true,
      "seed": 8,
      "target_accuracy": 0.1729,
      "time_to_recovery": 5
    },
    {
      "condition": "ONLINE_MANAGE_ONLY",
      "recovered": true,
      "seed": 9,
      "target_accuracy": 0.1653,
      "time_to_recovery": 5
    }
  ]
}