artifacts/metrics/omac_4b_trivia2code__factorial_summary.json

{
  "aurc_by_seed": [
    {
      "aurc": 0.1914,
      "condition": "FROZEN_EVOLVED",
      "mean_post_shift_accuracy": 0.1775,
      "n_post_shift": 479,
      "seed": 0
    },
    {
      "aurc": 0.2091,
      "condition": "FROZEN_EVOLVED",
      "mean_post_shift_accuracy": 0.2129,
      "n_post_shift": 479,
      "seed": 1
    },
    {
      "aurc": 0.1852,
      "condition": "FROZEN_EVOLVED",
      "mean_post_shift_accuracy": 0.1837,
      "n_post_shift": 479,
      "seed": 2
    },
    {
      "aurc": 0.2001,
      "condition": "FROZEN_EVOLVED",
      "mean_post_shift_accuracy": 0.1879,
      "n_post_shift": 479,
      "seed": 3
    },
    {
      "aurc": 0.154,
      "condition": "FROZEN_EVOLVED",
      "mean_post_shift_accuracy": 0.1691,
      "n_post_shift": 479,
      "seed": 4
    },
    {
      "aurc": 0.1875,
      "condition": "FROZEN_EVOLVED",
      "mean_post_shift_accuracy": 0.1712,
      "n_post_shift": 479,
      "seed": 5
    },
    {
      "aurc": 0.1914,
      "condition": "FROZEN_EVOLVED",
      "mean_post_shift_accuracy": 0.2004,
      "n_post_shift": 479,
      "seed": 6
    },
    {
      "aurc": 0.1984,
      "condition": "FROZEN_EVOLVED",
      "mean_post_shift_accuracy": 0.2129,
      "n_post_shift": 479,
      "seed": 7
    },
    {
      "aurc": 0.1711,
      "condition": "FROZEN_EVOLVED",
      "mean_post_shift_accuracy": 0.1691,
      "n_post_shift": 479,
      "seed": 8
    },
    {
      "aurc": 0.1627,
      "condition": "FROZEN_EVOLVED",
      "mean_post_shift_accuracy": 0.1858,
      "n_post_shift": 479,
      "seed": 9
    },
    {
      "aurc": 0.2367,
      "condition": "NO_MEMORY",
      "mean_post_shift_accuracy": 0.238,
      "n_post_shift": 479,
      "seed": 0
    },
    {
      "aurc": 0.248,
      "condition": "NO_MEMORY",
      "mean_post_shift_accuracy": 0.2401,
      "n_post_shift": 479,
      "seed": 1
    },
    {
      "aurc": 0.2275,
      "condition": "NO_MEMORY",
      "mean_post_shift_accuracy": 0.2401,
      "n_post_shift": 479,
      "seed": 2
    },
    {
      "aurc": 0.2599,
      "condition": "NO_MEMORY",
      "mean_post_shift_accuracy": 0.238,
      "n_post_shift": 479,
      "seed": 3
    },
    {
      "aurc": 0.2571,
      "condition": "NO_MEMORY",
      "mean_post_shift_accuracy": 0.2401,
      "n_post_shift": 479,
      "seed": 4
    },
    {
      "aurc": 0.2415,
      "condition": "NO_MEMORY",
      "mean_post_shift_accuracy": 0.2401,
      "n_post_shift": 479,
      "seed": 5
    },
    {
      "aurc": 0.2408,
      "condition": "NO_MEMORY",
      "mean_post_shift_accuracy": 0.2401,
      "n_post_shift": 479,
      "seed": 6
    },
    {
      "aurc": 0.247,
      "condition": "NO_MEMORY",
      "mean_post_shift_accuracy": 0.2401,
      "n_post_shift": 479,
      "seed": 7
    },
    {
      "aurc": 0.2541,
      "condition": "NO_MEMORY",
      "mean_post_shift_accuracy": 0.2401,
      "n_post_shift": 479,
      "seed": 8
    },
    {
      "aurc": 0.2215,
      "condition": "NO_MEMORY",
      "mean_post_shift_accuracy": 0.238,
      "n_post_shift": 479,
      "seed": 9
    },
    {
      "aurc": 0.1811,
      "condition": "ONLINE_JOINT",
      "mean_post_shift_accuracy": 0.19,
      "n_post_shift": 479,
      "seed": 0
    },
    {
      "aurc": 0.2119,
      "condition": "ONLINE_JOINT",
      "mean_post_shift_accuracy": 0.215,
      "n_post_shift": 479,
      "seed": 1
    },
    {
      "aurc": 0.188,
      "condition": "ONLINE_JOINT",
      "mean_post_shift_accuracy": 0.1942,
      "n_post_shift": 479,
      "seed": 2
    },
    {
      "aurc": 0.2148,
      "condition": "ONLINE_JOINT",
      "mean_post_shift_accuracy": 0.1962,
      "n_post_shift": 479,
      "seed": 3
    },
    {
      "aurc": 0.1546,
      "condition": "ONLINE_JOINT",
      "mean_post_shift_accuracy": 0.1649,
      "n_post_shift": 479,
      "seed": 4
    },
    {
      "aurc": 0.219,
      "condition": "ONLINE_JOINT",
      "mean_post_shift_accuracy": 0.1733,
      "n_post_shift": 479,
      "seed": 5
    },
    {
      "aurc": 0.1689,
      "condition": "ONLINE_JOINT",
      "mean_post_shift_accuracy": 0.19,
      "n_post_shift": 479,
      "seed": 6
    },
    {
      "aurc": 0.192,
      "condition": "ONLINE_JOINT",
      "mean_post_shift_accuracy": 0.2109,
      "n_post_shift": 479,
      "seed": 7
    },
    {
      "aurc": 0.2358,
      "condition": "ONLINE_JOINT",
      "mean_post_shift_accuracy": 0.2025,
      "n_post_shift": 479,
      "seed": 8
    },
    {
      "aurc": 0.1661,
      "condition": "ONLINE_JOINT",
      "mean_post_shift_accuracy": 0.1754,
      "n_post_shift": 479,
      "seed": 9
    },
    {
      "aurc": 0.1475,
      "condition": "ONLINE_MANAGE_ONLY",
      "mean_post_shift_accuracy": 0.1628,
      "n_post_shift": 479,
      "seed": 0
    },
    {
      "aurc": 0.2052,
      "condition": "ONLINE_MANAGE_ONLY",
      "mean_post_shift_accuracy": 0.19,
      "n_post_shift": 479,
      "seed": 1
    },
    {
      "aurc": 0.1882,
      "condition": "ONLINE_MANAGE_ONLY",
      "mean_post_shift_accuracy": 0.1942,
      "n_post_shift": 479,
      "seed": 2
    },
    {
      "aurc": 0.192,
      "condition": "ONLINE_MANAGE_ONLY",
      "mean_post_shift_accuracy": 0.1754,
      "n_post_shift": 479,
      "seed": 3
    },
    {
      "aurc": 0.1629,
      "condition": "ONLINE_MANAGE_ONLY",
      "mean_post_shift_accuracy": 0.1962,
      "n_post_shift": 479,
      "seed": 4
    },
    {
      "aurc": 0.1768,
      "condition": "ONLINE_MANAGE_ONLY",
      "mean_post_shift_accuracy": 0.1816,
      "n_post_shift": 479,
      "seed": 5
    },
    {
      "aurc": 0.1837,
      "condition": "ONLINE_MANAGE_ONLY",
      "mean_post_shift_accuracy": 0.2004,
      "n_post_shift": 479,
      "seed": 6
    },
    {
      "aurc": 0.1782,
      "condition": "ONLINE_MANAGE_ONLY",
      "mean_post_shift_accuracy": 0.1962,
      "n_post_shift": 479,
      "seed": 7
    },
    {
      "aurc": 0.2425,
      "condition": "ONLINE_MANAGE_ONLY",
      "mean_post_shift_accuracy": 0.2109,
      "n_post_shift": 479,
      "seed": 8
    },
    {
      "aurc": 0.1793,
      "condition": "ONLINE_MANAGE_ONLY",
      "mean_post_shift_accuracy": 0.2067,
      "n_post_shift": 479,
      "seed": 9
    }
  ],
  "conditions": {
    "FROZEN_EVOLVED": {
      "ci95_high": 0.1945,
      "ci95_low": 0.1746,
      "mean_aurc": 0.1851,
      "mean_exploration_rate": 0.0,
      "mean_ttr": 19.1,
      "n_seeds": 10,
      "recovery_rate": 1.0
    },
    "NO_MEMORY": {
      "ci95_high": 0.2501,
      "ci95_low": 0.2359,
      "mean_aurc": 0.2434,
      "mean_exploration_rate": 0.0,
      "mean_ttr": 7.9,
      "n_seeds": 10,
      "recovery_rate": 1.0
    },
    "ONLINE_JOINT": {
      "ci95_high": 0.208,
      "ci95_low": 0.178,
      "mean_aurc": 0.1932,
      "mean_exploration_rate": 1.0,
      "mean_ttr": 16.4,
      "n_seeds": 10,
      "recovery_rate": 1.0
    },
    "ONLINE_MANAGE_ONLY": {
      "ci95_high": 0.2021,
      "ci95_low": 0.1708,
      "mean_aurc": 0.1856,
      "mean_exploration_rate": 0.9452,
      "mean_ttr": 23.2,
      "n_seeds": 10,
      "recovery_rate": 1.0
    }
  },
  "n_blocks_total": 3920,
  "n_episodes_total": 39160,
  "shift_episode": 500,
  "ttr_by_seed": [
    {
      "condition": "FROZEN_EVOLVED",
      "recovered": true,
      "seed": 0,
      "target_accuracy": 0.3654,
      "time_to_recovery": 6
    },
    {
      "condition": "FROZEN_EVOLVED",
      "recovered": true,
      "seed": 1,
      "target_accuracy": 0.3636,
      "time_to_recovery": 21
    },
    {
      "condition": "FROZEN_EVOLVED",
      "recovered": true,
      "seed": 2,
      "target_accuracy": 0.3744,
      "time_to_recovery": 5
    },
    {
      "condition": "FROZEN_EVOLVED",
      "recovered": true,
      "seed": 3,
      "target_accuracy": 0.3564,
      "time_to_recovery": 22
    },
    {
      "condition": "FROZEN_EVOLVED",
      "recovered": true,
      "seed": 4,
      "target_accuracy": 0.3546,
      "time_to_recovery": 22
    },
    {
      "condition": "FROZEN_EVOLVED",
      "recovered": true,
      "seed": 5,
      "target_accuracy": 0.3636,
      "time_to_recovery": 13
    },
    {
      "condition": "FROZEN_EVOLVED",
      "recovered": true,
      "seed": 6,
      "target_accuracy": 0.3546,
      "time_to_recovery": 15
    },
    {
      "condition": "FROZEN_EVOLVED",
      "recovered": true,
      "seed": 7,
      "target_accuracy": 0.3636,
      "time_to_recovery": 31
    },
    {
      "condition": "FROZEN_EVOLVED",
      "recovered": true,
      "seed": 8,
      "target_accuracy": 0.3654,
      "time_to_recovery": 32
    },
    {
      "condition": "FROZEN_EVOLVED",
      "recovered": true,
      "seed": 9,
      "target_accuracy": 0.3618,
      "time_to_recovery": 24
    },
    {
      "condition": "NO_MEMORY",
      "recovered": true,
      "seed": 0,
      "target_accuracy": 0.3726,
      "time_to_recovery": 6
    },
    {
      "condition": "NO_MEMORY",
      "recovered": true,
      "seed": 1,
      "target_accuracy": 0.3726,
      "time_to_recovery": 5
    },
    {
      "condition": "NO_MEMORY",
      "recovered": true,
      "seed": 2,
      "target_accuracy": 0.3744,
      "time_to_recovery": 8
    },
    {
      "condition": "NO_MEMORY",
      "recovered": true,
      "seed": 3,
      "target_accuracy": 0.3726,
      "time_to_recovery": 5
    },
    {
      "condition": "NO_MEMORY",
      "recovered": true,
      "seed": 4,
      "target_accuracy": 0.3726,
      "time_to_recovery": 5
    },
    {
      "condition": "NO_MEMORY",
      "recovered": true,
      "seed": 5,
      "target_accuracy": 0.3726,
      "time_to_recovery": 7
    },
    {
      "condition": "NO_MEMORY",
      "recovered": true,
      "seed": 6,
      "target_accuracy": 0.3726,
      "time_to_recovery": 5
    },
    {
      "condition": "NO_MEMORY",
      "recovered": true,
      "seed": 7,
      "target_accuracy": 0.3726,
      "time_to_recovery": 5
    },
    {
      "condition": "NO_MEMORY",
      "recovered": true,
      "seed": 8,
      "target_accuracy": 0.3726,
      "time_to_recovery": 5
    },
    {
      "condition": "NO_MEMORY",
      "recovered": true,
      "seed": 9,
      "target_accuracy": 0.3726,
      "time_to_recovery": 28
    },
    {
      "condition": "ONLINE_JOINT",
      "recovered": true,
      "seed": 0,
      "target_accuracy": 0.3672,
      "time_to_recovery": 6
    },
    {
      "condition": "ONLINE_JOINT",
      "recovered": true,
      "seed": 1,
      "target_accuracy": 0.3528,
      "time_to_recovery": 34
    },
    {
      "condition": "ONLINE_JOINT",
      "recovered": true,
      "seed": 2,
      "target_accuracy": 0.369,
      "time_to_recovery": 22
    },
    {
      "condition": "ONLINE_JOINT",
      "recovered": true,
      "seed": 3,
      "target_accuracy": 0.3636,
      "time_to_recovery": 5
    },
    {
      "condition": "ONLINE_JOINT",
      "recovered": true,
      "seed": 4,
      "target_accuracy": 0.3726,
      "time_to_recovery": 6
    },
    {
      "condition": "ONLINE_JOINT",
      "recovered": true,
      "seed": 5,
      "target_accuracy": 0.3528,
      "time_to_recovery": 7
    },
    {
      "condition": "ONLINE_JOINT",
      "recovered": true,
      "seed": 6,
      "target_accuracy": 0.3582,
      "time_to_recovery": 19
    },
    {
      "condition": "ONLINE_JOINT",
      "recovered": true,
      "seed": 7,
      "target_accuracy": 0.36,
      "time_to_recovery": 36
    },
    {
      "condition": "ONLINE_JOINT",
      "recovered": true,
      "seed": 8,
      "target_accuracy": 0.3618,
      "time_to_recovery": 5
    },
    {
      "condition": "ONLINE_JOINT",
      "recovered": true,
      "seed": 9,
      "target_accuracy": 0.3636,
      "time_to_recovery": 24
    },
    {
      "condition": "ONLINE_MANAGE_ONLY",
      "recovered": true,
      "seed": 0,
      "target_accuracy": 0.3618,
      "time_to_recovery": 6
    },
    {
      "condition": "ONLINE_MANAGE_ONLY",
      "recovered": true,
      "seed": 1,
      "target_accuracy": 0.3618,
      "time_to_recovery": 34
    },
    {
      "condition": "ONLINE_MANAGE_ONLY",
      "recovered": true,
      "seed": 2,
      "target_accuracy": 0.3708,
      "time_to_recovery": 23
    },
    {
      "condition": "ONLINE_MANAGE_ONLY",
      "recovered": true,
      "seed": 3,
      "target_accuracy": 0.3546,
      "time_to_recovery": 5
    },
    {
      "condition": "ONLINE_MANAGE_ONLY",
      "recovered": true,
      "seed": 4,
      "target_accuracy": 0.3618,
      "time_to_recovery": 43
    },
    {
      "condition": "ONLINE_MANAGE_ONLY",
      "recovered": true,
      "seed": 5,
      "target_accuracy": 0.3564,
      "time_to_recovery": 9
    },
    {
      "condition": "ONLINE_MANAGE_ONLY",
      "recovered": true,
      "seed": 6,
      "target_accuracy": 0.3726,
      "time_to_recovery": 15
    },
    {
      "condition": "ONLINE_MANAGE_ONLY",
      "recovered": true,
      "seed": 7,
      "target_accuracy": 0.369,
      "time_to_recovery": 36
    },
    {
      "condition": "ONLINE_MANAGE_ONLY",
      "recovered": true,
      "seed": 8,
      "target_accuracy": 0.3546,
      "time_to_recovery": 17
    },
    {
      "condition": "ONLINE_MANAGE_ONLY",
      "recovered": true,
      "seed": 9,
      "target_accuracy": 0.3618,
      "time_to_recovery": 44
    }
  ]
}