artifacts/metrics/omac_4b_sci2code__factorial_summary.json

{
  "aurc_by_seed": [
    {
      "aurc": 0.1744,
      "condition": "FROZEN_EVOLVED",
      "mean_post_shift_accuracy": 0.1858,
      "n_post_shift": 479,
      "seed": 0
    },
    {
      "aurc": 0.1727,
      "condition": "FROZEN_EVOLVED",
      "mean_post_shift_accuracy": 0.1816,
      "n_post_shift": 479,
      "seed": 1
    },
    {
      "aurc": 0.1934,
      "condition": "FROZEN_EVOLVED",
      "mean_post_shift_accuracy": 0.2088,
      "n_post_shift": 479,
      "seed": 2
    },
    {
      "aurc": 0.2158,
      "condition": "FROZEN_EVOLVED",
      "mean_post_shift_accuracy": 0.1983,
      "n_post_shift": 479,
      "seed": 3
    },
    {
      "aurc": 0.1784,
      "condition": "FROZEN_EVOLVED",
      "mean_post_shift_accuracy": 0.1858,
      "n_post_shift": 479,
      "seed": 4
    },
    {
      "aurc": 0.2035,
      "condition": "FROZEN_EVOLVED",
      "mean_post_shift_accuracy": 0.1921,
      "n_post_shift": 479,
      "seed": 5
    },
    {
      "aurc": 0.1793,
      "condition": "FROZEN_EVOLVED",
      "mean_post_shift_accuracy": 0.2025,
      "n_post_shift": 479,
      "seed": 6
    },
    {
      "aurc": 0.1865,
      "condition": "FROZEN_EVOLVED",
      "mean_post_shift_accuracy": 0.1921,
      "n_post_shift": 479,
      "seed": 7
    },
    {
      "aurc": 0.2221,
      "condition": "FROZEN_EVOLVED",
      "mean_post_shift_accuracy": 0.1962,
      "n_post_shift": 479,
      "seed": 8
    },
    {
      "aurc": 0.1947,
      "condition": "FROZEN_EVOLVED",
      "mean_post_shift_accuracy": 0.2046,
      "n_post_shift": 479,
      "seed": 9
    },
    {
      "aurc": 0.2425,
      "condition": "NO_MEMORY",
      "mean_post_shift_accuracy": 0.2359,
      "n_post_shift": 479,
      "seed": 0
    },
    {
      "aurc": 0.246,
      "condition": "NO_MEMORY",
      "mean_post_shift_accuracy": 0.2359,
      "n_post_shift": 479,
      "seed": 1
    },
    {
      "aurc": 0.2275,
      "condition": "NO_MEMORY",
      "mean_post_shift_accuracy": 0.2401,
      "n_post_shift": 479,
      "seed": 2
    },
    {
      "aurc": 0.2618,
      "condition": "NO_MEMORY",
      "mean_post_shift_accuracy": 0.2401,
      "n_post_shift": 479,
      "seed": 3
    },
    {
      "aurc": 0.2571,
      "condition": "NO_MEMORY",
      "mean_post_shift_accuracy": 0.238,
      "n_post_shift": 479,
      "seed": 4
    },
    {
      "aurc": 0.2415,
      "condition": "NO_MEMORY",
      "mean_post_shift_accuracy": 0.2401,
      "n_post_shift": 479,
      "seed": 5
    },
    {
      "aurc": 0.2408,
      "condition": "NO_MEMORY",
      "mean_post_shift_accuracy": 0.2401,
      "n_post_shift": 479,
      "seed": 6
    },
    {
      "aurc": 0.2457,
      "condition": "NO_MEMORY",
      "mean_post_shift_accuracy": 0.238,
      "n_post_shift": 479,
      "seed": 7
    },
    {
      "aurc": 0.2541,
      "condition": "NO_MEMORY",
      "mean_post_shift_accuracy": 0.2401,
      "n_post_shift": 479,
      "seed": 8
    },
    {
      "aurc": 0.2222,
      "condition": "NO_MEMORY",
      "mean_post_shift_accuracy": 0.238,
      "n_post_shift": 479,
      "seed": 9
    },
    {
      "aurc": 0.1835,
      "condition": "ONLINE_JOINT",
      "mean_post_shift_accuracy": 0.1858,
      "n_post_shift": 479,
      "seed": 0
    },
    {
      "aurc": 0.194,
      "condition": "ONLINE_JOINT",
      "mean_post_shift_accuracy": 0.2046,
      "n_post_shift": 479,
      "seed": 1
    },
    {
      "aurc": 0.2167,
      "condition": "ONLINE_JOINT",
      "mean_post_shift_accuracy": 0.2067,
      "n_post_shift": 479,
      "seed": 2
    },
    {
      "aurc": 0.1781,
      "condition": "ONLINE_JOINT",
      "mean_post_shift_accuracy": 0.1775,
      "n_post_shift": 479,
      "seed": 3
    },
    {
      "aurc": 0.1645,
      "condition": "ONLINE_JOINT",
      "mean_post_shift_accuracy": 0.1879,
      "n_post_shift": 479,
      "seed": 4
    },
    {
      "aurc": 0.2223,
      "condition": "ONLINE_JOINT",
      "mean_post_shift_accuracy": 0.2025,
      "n_post_shift": 479,
      "seed": 5
    },
    {
      "aurc": 0.1913,
      "condition": "ONLINE_JOINT",
      "mean_post_shift_accuracy": 0.2046,
      "n_post_shift": 479,
      "seed": 6
    },
    {
      "aurc": 0.1976,
      "condition": "ONLINE_JOINT",
      "mean_post_shift_accuracy": 0.2046,
      "n_post_shift": 479,
      "seed": 7
    },
    {
      "aurc": 0.217,
      "condition": "ONLINE_JOINT",
      "mean_post_shift_accuracy": 0.1587,
      "n_post_shift": 479,
      "seed": 8
    },
    {
      "aurc": 0.1746,
      "condition": "ONLINE_JOINT",
      "mean_post_shift_accuracy": 0.2004,
      "n_post_shift": 479,
      "seed": 9
    },
    {
      "aurc": 0.1859,
      "condition": "ONLINE_MANAGE_ONLY",
      "mean_post_shift_accuracy": 0.19,
      "n_post_shift": 479,
      "seed": 0
    },
    {
      "aurc": 0.1982,
      "condition": "ONLINE_MANAGE_ONLY",
      "mean_post_shift_accuracy": 0.1962,
      "n_post_shift": 479,
      "seed": 1
    },
    {
      "aurc": 0.1801,
      "condition": "ONLINE_MANAGE_ONLY",
      "mean_post_shift_accuracy": 0.2025,
      "n_post_shift": 479,
      "seed": 2
    },
    {
      "aurc": 0.2169,
      "condition": "ONLINE_MANAGE_ONLY",
      "mean_post_shift_accuracy": 0.2171,
      "n_post_shift": 479,
      "seed": 3
    },
    {
      "aurc": 0.1702,
      "condition": "ONLINE_MANAGE_ONLY",
      "mean_post_shift_accuracy": 0.1712,
      "n_post_shift": 479,
      "seed": 4
    },
    {
      "aurc": 0.1993,
      "condition": "ONLINE_MANAGE_ONLY",
      "mean_post_shift_accuracy": 0.19,
      "n_post_shift": 479,
      "seed": 5
    },
    {
      "aurc": 0.178,
      "condition": "ONLINE_MANAGE_ONLY",
      "mean_post_shift_accuracy": 0.1942,
      "n_post_shift": 479,
      "seed": 6
    },
    {
      "aurc": 0.186,
      "condition": "ONLINE_MANAGE_ONLY",
      "mean_post_shift_accuracy": 0.19,
      "n_post_shift": 479,
      "seed": 7
    },
    {
      "aurc": 0.2087,
      "condition": "ONLINE_MANAGE_ONLY",
      "mean_post_shift_accuracy": 0.1879,
      "n_post_shift": 479,
      "seed": 8
    },
    {
      "aurc": 0.2006,
      "condition": "ONLINE_MANAGE_ONLY",
      "mean_post_shift_accuracy": 0.215,
      "n_post_shift": 479,
      "seed": 9
    }
  ],
  "conditions": {
    "FROZEN_EVOLVED": {
      "ci95_high": 0.2029,
      "ci95_low": 0.1825,
      "mean_aurc": 0.1921,
      "mean_exploration_rate": 0.0,
      "mean_ttr": 166.4,
      "n_seeds": 10,
      "recovery_rate": 0.5
    },
    "NO_MEMORY": {
      "ci95_high": 0.2507,
      "ci95_low": 0.2366,
      "mean_aurc": 0.2439,
      "mean_exploration_rate": 0.0,
      "mean_ttr": 192.7,
      "n_seeds": 10,
      "recovery_rate": 1.0
    },
    "ONLINE_JOINT": {
      "ci95_high": 0.2053,
      "ci95_low": 0.1819,
      "mean_aurc": 0.194,
      "mean_exploration_rate": 1.0,
      "mean_ttr": 157.2,
      "n_seeds": 10,
      "recovery_rate": 0.6
    },
    "ONLINE_MANAGE_ONLY": {
      "ci95_high": 0.2015,
      "ci95_low": 0.1837,
      "mean_aurc": 0.1924,
      "mean_exploration_rate": 0.9453,
      "mean_ttr": 179.0,
      "n_seeds": 10,
      "recovery_rate": 0.6
    }
  },
  "n_blocks_total": 3920,
  "n_episodes_total": 39160,
  "shift_episode": 500,
  "ttr_by_seed": [
    {
      "condition": "FROZEN_EVOLVED",
      "recovered": false,
      "seed": 0,
      "target_accuracy": 0.8028,
      "time_to_recovery": null
    },
    {
      "condition": "FROZEN_EVOLVED",
      "recovered": false,
      "seed": 1,
      "target_accuracy": 0.801,
      "time_to_recovery": null
    },
    {
      "condition": "FROZEN_EVOLVED",
      "recovered": false,
      "seed": 2,
      "target_accuracy": 0.801,
      "time_to_recovery": null
    },
    {
      "condition": "FROZEN_EVOLVED",
      "recovered": true,
      "seed": 3,
      "target_accuracy": 0.7884,
      "time_to_recovery": 72
    },
    {
      "condition": "FROZEN_EVOLVED",
      "recovered": true,
      "seed": 4,
      "target_accuracy": 0.7938,
      "time_to_recovery": 147
    },
    {
      "condition": "FROZEN_EVOLVED",
      "recovered": true,
      "seed": 5,
      "target_accuracy": 0.7938,
      "time_to_recovery": 158
    },
    {
      "condition": "FROZEN_EVOLVED",
      "recovered": false,
      "seed": 6,
      "target_accuracy": 0.7974,
      "time_to_recovery": null
    },
    {
      "condition": "FROZEN_EVOLVED",
      "recovered": true,
      "seed": 7,
      "target_accuracy": 0.7884,
      "time_to_recovery": 158
    },
    {
      "condition": "FROZEN_EVOLVED",
      "recovered": true,
      "seed": 8,
      "target_accuracy": 0.7902,
      "time_to_recovery": 297
    },
    {
      "condition": "FROZEN_EVOLVED",
      "recovered": false,
      "seed": 9,
      "target_accuracy": 0.7956,
      "time_to_recovery": null
    },
    {
      "condition": "NO_MEMORY",
      "recovered": true,
      "seed": 0,
      "target_accuracy": 0.7866,
      "time_to_recovery": 10
    },
    {
      "condition": "NO_MEMORY",
      "recovered": true,
      "seed": 1,
      "target_accuracy": 0.7848,
      "time_to_recovery": 376
    },
    {
      "condition": "NO_MEMORY",
      "recovered": true,
      "seed": 2,
      "target_accuracy": 0.7866,
      "time_to_recovery": 342
    },
    {
      "condition": "NO_MEMORY",
      "recovered": true,
      "seed": 3,
      "target_accuracy": 0.7866,
      "time_to_recovery": 119
    },
    {
      "condition": "NO_MEMORY",
      "recovered": true,
      "seed": 4,
      "target_accuracy": 0.7866,
      "time_to_recovery": 147
    },
    {
      "condition": "NO_MEMORY",
      "recovered": true,
      "seed": 5,
      "target_accuracy": 0.7884,
      "time_to_recovery": 368
    },
    {
      "condition": "NO_MEMORY",
      "recovered": true,
      "seed": 6,
      "target_accuracy": 0.7866,
      "time_to_recovery": 232
    },
    {
      "condition": "NO_MEMORY",
      "recovered": true,
      "seed": 7,
      "target_accuracy": 0.7866,
      "time_to_recovery": 39
    },
    {
      "condition": "NO_MEMORY",
      "recovered": true,
      "seed": 8,
      "target_accuracy": 0.7866,
      "time_to_recovery": 78
    },
    {
      "condition": "NO_MEMORY",
      "recovered": true,
      "seed": 9,
      "target_accuracy": 0.7866,
      "time_to_recovery": 216
    },
    {
      "condition": "ONLINE_JOINT",
      "recovered": true,
      "seed": 0,
      "target_accuracy": 0.7974,
      "time_to_recovery": 123
    },
    {
      "condition": "ONLINE_JOINT",
      "recovered": false,
      "seed": 1,
      "target_accuracy": 0.801,
      "time_to_recovery": null
    },
    {
      "condition": "ONLINE_JOINT",
      "recovered": false,
      "seed": 2,
      "target_accuracy": 0.8046,
      "time_to_recovery": null
    },
    {
      "condition": "ONLINE_JOINT",
      "recovered": true,
      "seed": 3,
      "target_accuracy": 0.7938,
      "time_to_recovery": 72
    },
    {
      "condition": "ONLINE_JOINT",
      "recovered": true,
      "seed": 4,
      "target_accuracy": 0.7938,
      "time_to_recovery": 147
    },
    {
      "condition": "ONLINE_JOINT",
      "recovered": false,
      "seed": 5,
      "target_accuracy": 0.792,
      "time_to_recovery": null
    },
    {
      "condition": "ONLINE_JOINT",
      "recovered": true,
      "seed": 6,
      "target_accuracy": 0.7848,
      "time_to_recovery": 199
    },
    {
      "condition": "ONLINE_JOINT",
      "recovered": true,
      "seed": 7,
      "target_accuracy": 0.792,
      "time_to_recovery": 262
    },
    {
      "condition": "ONLINE_JOINT",
      "recovered": false,
      "seed": 8,
      "target_accuracy": 0.7902,
      "time_to_recovery": null
    },
    {
      "condition": "ONLINE_JOINT",
      "recovered": true,
      "seed": 9,
      "target_accuracy": 0.7938,
      "time_to_recovery": 140
    },
    {
      "condition": "ONLINE_MANAGE_ONLY",
      "recovered": false,
      "seed": 0,
      "target_accuracy": 0.8046,
      "time_to_recovery": null
    },
    {
      "condition": "ONLINE_MANAGE_ONLY",
      "recovered": false,
      "seed": 1,
      "target_accuracy": 0.801,
      "time_to_recovery": null
    },
    {
      "condition": "ONLINE_MANAGE_ONLY",
      "recovered": false,
      "seed": 2,
      "target_accuracy": 0.801,
      "time_to_recovery": null
    },
    {
      "condition": "ONLINE_MANAGE_ONLY",
      "recovered": true,
      "seed": 3,
      "target_accuracy": 0.7902,
      "time_to_recovery": 72
    },
    {
      "condition": "ONLINE_MANAGE_ONLY",
      "recovered": true,
      "seed": 4,
      "target_accuracy": 0.7902,
      "time_to_recovery": 147
    },
    {
      "condition": "ONLINE_MANAGE_ONLY",
      "recovered": true,
      "seed": 5,
      "target_accuracy": 0.7956,
      "time_to_recovery": 158
    },
    {
      "condition": "ONLINE_MANAGE_ONLY",
      "recovered": true,
      "seed": 6,
      "target_accuracy": 0.7956,
      "time_to_recovery": 459
    },
    {
      "condition": "ONLINE_MANAGE_ONLY",
      "recovered": true,
      "seed": 7,
      "target_accuracy": 0.7884,
      "time_to_recovery": 158
    },
    {
      "condition": "ONLINE_MANAGE_ONLY",
      "recovered": true,
      "seed": 8,
      "target_accuracy": 0.792,
      "time_to_recovery": 80
    },
    {
      "condition": "ONLINE_MANAGE_ONLY",
      "recovered": false,
      "seed": 9,
      "target_accuracy": 0.7956,
      "time_to_recovery": null
    }
  ]
}