artifacts/metrics/omac_4b_code2sci__factorial_summary.json

{
  "aurc_by_seed": [
    {
      "aurc": 0.8796,
      "condition": "FROZEN_EVOLVED",
      "mean_post_shift_accuracy": 0.886,
      "n_post_shift": 500,
      "seed": 0
    },
    {
      "aurc": 0.9065,
      "condition": "FROZEN_EVOLVED",
      "mean_post_shift_accuracy": 0.882,
      "n_post_shift": 500,
      "seed": 1
    },
    {
      "aurc": 0.8712,
      "condition": "FROZEN_EVOLVED",
      "mean_post_shift_accuracy": 0.872,
      "n_post_shift": 500,
      "seed": 2
    },
    {
      "aurc": 0.8486,
      "condition": "FROZEN_EVOLVED",
      "mean_post_shift_accuracy": 0.872,
      "n_post_shift": 500,
      "seed": 3
    },
    {
      "aurc": 0.8738,
      "condition": "FROZEN_EVOLVED",
      "mean_post_shift_accuracy": 0.884,
      "n_post_shift": 500,
      "seed": 4
    },
    {
      "aurc": 0.8696,
      "condition": "FROZEN_EVOLVED",
      "mean_post_shift_accuracy": 0.866,
      "n_post_shift": 500,
      "seed": 5
    },
    {
      "aurc": 0.8677,
      "condition": "FROZEN_EVOLVED",
      "mean_post_shift_accuracy": 0.854,
      "n_post_shift": 500,
      "seed": 6
    },
    {
      "aurc": 0.8484,
      "condition": "FROZEN_EVOLVED",
      "mean_post_shift_accuracy": 0.882,
      "n_post_shift": 500,
      "seed": 7
    },
    {
      "aurc": 0.8721,
      "condition": "FROZEN_EVOLVED",
      "mean_post_shift_accuracy": 0.878,
      "n_post_shift": 500,
      "seed": 8
    },
    {
      "aurc": 0.8588,
      "condition": "FROZEN_EVOLVED",
      "mean_post_shift_accuracy": 0.874,
      "n_post_shift": 500,
      "seed": 9
    },
    {
      "aurc": 0.8553,
      "condition": "NO_MEMORY",
      "mean_post_shift_accuracy": 0.874,
      "n_post_shift": 500,
      "seed": 0
    },
    {
      "aurc": 0.8908,
      "condition": "NO_MEMORY",
      "mean_post_shift_accuracy": 0.874,
      "n_post_shift": 500,
      "seed": 1
    },
    {
      "aurc": 0.8774,
      "condition": "NO_MEMORY",
      "mean_post_shift_accuracy": 0.874,
      "n_post_shift": 500,
      "seed": 2
    },
    {
      "aurc": 0.8434,
      "condition": "NO_MEMORY",
      "mean_post_shift_accuracy": 0.874,
      "n_post_shift": 500,
      "seed": 3
    },
    {
      "aurc": 0.8788,
      "condition": "NO_MEMORY",
      "mean_post_shift_accuracy": 0.874,
      "n_post_shift": 500,
      "seed": 4
    },
    {
      "aurc": 0.8793,
      "condition": "NO_MEMORY",
      "mean_post_shift_accuracy": 0.872,
      "n_post_shift": 500,
      "seed": 5
    },
    {
      "aurc": 0.8998,
      "condition": "NO_MEMORY",
      "mean_post_shift_accuracy": 0.874,
      "n_post_shift": 500,
      "seed": 6
    },
    {
      "aurc": 0.8596,
      "condition": "NO_MEMORY",
      "mean_post_shift_accuracy": 0.874,
      "n_post_shift": 500,
      "seed": 7
    },
    {
      "aurc": 0.8574,
      "condition": "NO_MEMORY",
      "mean_post_shift_accuracy": 0.874,
      "n_post_shift": 500,
      "seed": 8
    },
    {
      "aurc": 0.8795,
      "condition": "NO_MEMORY",
      "mean_post_shift_accuracy": 0.874,
      "n_post_shift": 500,
      "seed": 9
    },
    {
      "aurc": 0.8211,
      "condition": "ONLINE_JOINT",
      "mean_post_shift_accuracy": 0.858,
      "n_post_shift": 500,
      "seed": 0
    },
    {
      "aurc": 0.9032,
      "condition": "ONLINE_JOINT",
      "mean_post_shift_accuracy": 0.88,
      "n_post_shift": 500,
      "seed": 1
    },
    {
      "aurc": 0.8771,
      "condition": "ONLINE_JOINT",
      "mean_post_shift_accuracy": 0.884,
      "n_post_shift": 500,
      "seed": 2
    },
    {
      "aurc": 0.8617,
      "condition": "ONLINE_JOINT",
      "mean_post_shift_accuracy": 0.878,
      "n_post_shift": 500,
      "seed": 3
    },
    {
      "aurc": 0.8859,
      "condition": "ONLINE_JOINT",
      "mean_post_shift_accuracy": 0.882,
      "n_post_shift": 500,
      "seed": 4
    },
    {
      "aurc": 0.8806,
      "condition": "ONLINE_JOINT",
      "mean_post_shift_accuracy": 0.884,
      "n_post_shift": 500,
      "seed": 5
    },
    {
      "aurc": 0.8977,
      "condition": "ONLINE_JOINT",
      "mean_post_shift_accuracy": 0.868,
      "n_post_shift": 500,
      "seed": 6
    },
    {
      "aurc": 0.8599,
      "condition": "ONLINE_JOINT",
      "mean_post_shift_accuracy": 0.874,
      "n_post_shift": 500,
      "seed": 7
    },
    {
      "aurc": 0.8833,
      "condition": "ONLINE_JOINT",
      "mean_post_shift_accuracy": 0.89,
      "n_post_shift": 500,
      "seed": 8
    },
    {
      "aurc": 0.8776,
      "condition": "ONLINE_JOINT",
      "mean_post_shift_accuracy": 0.88,
      "n_post_shift": 500,
      "seed": 9
    },
    {
      "aurc": 0.8787,
      "condition": "ONLINE_MANAGE_ONLY",
      "mean_post_shift_accuracy": 0.884,
      "n_post_shift": 500,
      "seed": 0
    },
    {
      "aurc": 0.9064,
      "condition": "ONLINE_MANAGE_ONLY",
      "mean_post_shift_accuracy": 0.882,
      "n_post_shift": 500,
      "seed": 1
    },
    {
      "aurc": 0.8664,
      "condition": "ONLINE_MANAGE_ONLY",
      "mean_post_shift_accuracy": 0.87,
      "n_post_shift": 500,
      "seed": 2
    },
    {
      "aurc": 0.8508,
      "condition": "ONLINE_MANAGE_ONLY",
      "mean_post_shift_accuracy": 0.876,
      "n_post_shift": 500,
      "seed": 3
    },
    {
      "aurc": 0.8577,
      "condition": "ONLINE_MANAGE_ONLY",
      "mean_post_shift_accuracy": 0.88,
      "n_post_shift": 500,
      "seed": 4
    },
    {
      "aurc": 0.8696,
      "condition": "ONLINE_MANAGE_ONLY",
      "mean_post_shift_accuracy": 0.866,
      "n_post_shift": 500,
      "seed": 5
    },
    {
      "aurc": 0.8976,
      "condition": "ONLINE_MANAGE_ONLY",
      "mean_post_shift_accuracy": 0.87,
      "n_post_shift": 500,
      "seed": 6
    },
    {
      "aurc": 0.8252,
      "condition": "ONLINE_MANAGE_ONLY",
      "mean_post_shift_accuracy": 0.866,
      "n_post_shift": 500,
      "seed": 7
    },
    {
      "aurc": 0.8721,
      "condition": "ONLINE_MANAGE_ONLY",
      "mean_post_shift_accuracy": 0.878,
      "n_post_shift": 500,
      "seed": 8
    },
    {
      "aurc": 0.8806,
      "condition": "ONLINE_MANAGE_ONLY",
      "mean_post_shift_accuracy": 0.88,
      "n_post_shift": 500,
      "seed": 9
    }
  ],
  "conditions": {
    "FROZEN_EVOLVED": {
      "ci95_high": 0.8803,
      "ci95_low": 0.8609,
      "mean_aurc": 0.8696,
      "mean_exploration_rate": 0.0,
      "mean_ttr": 5.0,
      "n_seeds": 10,
      "recovery_rate": 1.0
    },
    "NO_MEMORY": {
      "ci95_high": 0.8827,
      "ci95_low": 0.8609,
      "mean_aurc": 0.8721,
      "mean_exploration_rate": 0.0,
      "mean_ttr": 5.0,
      "n_seeds": 10,
      "recovery_rate": 1.0
    },
    "ONLINE_JOINT": {
      "ci95_high": 0.8866,
      "ci95_low": 0.8602,
      "mean_aurc": 0.8748,
      "mean_exploration_rate": 1.0,
      "mean_ttr": 5.0,
      "n_seeds": 10,
      "recovery_rate": 1.0
    },
    "ONLINE_MANAGE_ONLY": {
      "ci95_high": 0.884,
      "ci95_low": 0.8565,
      "mean_aurc": 0.8705,
      "mean_exploration_rate": 0.9437,
      "mean_ttr": 5.0,
      "n_seeds": 10,
      "recovery_rate": 1.0
    }
  },
  "n_blocks_total": 3920,
  "n_episodes_total": 39160,
  "shift_episode": 479,
  "ttr_by_seed": [
    {
      "condition": "FROZEN_EVOLVED",
      "recovered": true,
      "seed": 0,
      "target_accuracy": 0.1578,
      "time_to_recovery": 5
    },
    {
      "condition": "FROZEN_EVOLVED",
      "recovered": true,
      "seed": 1,
      "target_accuracy": 0.1841,
      "time_to_recovery": 5
    },
    {
      "condition": "FROZEN_EVOLVED",
      "recovered": true,
      "seed": 2,
      "target_accuracy": 0.1785,
      "time_to_recovery": 5
    },
    {
      "condition": "FROZEN_EVOLVED",
      "recovered": true,
      "seed": 3,
      "target_accuracy": 0.171,
      "time_to_recovery": 5
    },
    {
      "condition": "FROZEN_EVOLVED",
      "recovered": true,
      "seed": 4,
      "target_accuracy": 0.1691,
      "time_to_recovery": 5
    },
    {
      "condition": "FROZEN_EVOLVED",
      "recovered": true,
      "seed": 5,
      "target_accuracy": 0.1916,
      "time_to_recovery": 5
    },
    {
      "condition": "FROZEN_EVOLVED",
      "recovered": true,
      "seed": 6,
      "target_accuracy": 0.1522,
      "time_to_recovery": 5
    },
    {
      "condition": "FROZEN_EVOLVED",
      "recovered": true,
      "seed": 7,
      "target_accuracy": 0.1691,
      "time_to_recovery": 5
    },
    {
      "condition": "FROZEN_EVOLVED",
      "recovered": true,
      "seed": 8,
      "target_accuracy": 0.1785,
      "time_to_recovery": 5
    },
    {
      "condition": "FROZEN_EVOLVED",
      "recovered": true,
      "seed": 9,
      "target_accuracy": 0.1503,
      "time_to_recovery": 5
    },
    {
      "condition": "NO_MEMORY",
      "recovered": true,
      "seed": 0,
      "target_accuracy": 0.2142,
      "time_to_recovery": 5
    },
    {
      "condition": "NO_MEMORY",
      "recovered": true,
      "seed": 1,
      "target_accuracy": 0.2161,
      "time_to_recovery": 5
    },
    {
      "condition": "NO_MEMORY",
      "recovered": true,
      "seed": 2,
      "target_accuracy": 0.2161,
      "time_to_recovery": 5
    },
    {
      "condition": "NO_MEMORY",
      "recovered": true,
      "seed": 3,
      "target_accuracy": 0.2161,
      "time_to_recovery": 5
    },
    {
      "condition": "NO_MEMORY",
      "recovered": true,
      "seed": 4,
      "target_accuracy": 0.2161,
      "time_to_recovery": 5
    },
    {
      "condition": "NO_MEMORY",
      "recovered": true,
      "seed": 5,
      "target_accuracy": 0.2142,
      "time_to_recovery": 5
    },
    {
      "condition": "NO_MEMORY",
      "recovered": true,
      "seed": 6,
      "target_accuracy": 0.2161,
      "time_to_recovery": 5
    },
    {
      "condition": "NO_MEMORY",
      "recovered": true,
      "seed": 7,
      "target_accuracy": 0.2142,
      "time_to_recovery": 5
    },
    {
      "condition": "NO_MEMORY",
      "recovered": true,
      "seed": 8,
      "target_accuracy": 0.2161,
      "time_to_recovery": 5
    },
    {
      "condition": "NO_MEMORY",
      "recovered": true,
      "seed": 9,
      "target_accuracy": 0.2123,
      "time_to_recovery": 5
    },
    {
      "condition": "ONLINE_JOINT",
      "recovered": true,
      "seed": 0,
      "target_accuracy": 0.1747,
      "time_to_recovery": 5
    },
    {
      "condition": "ONLINE_JOINT",
      "recovered": true,
      "seed": 1,
      "target_accuracy": 0.1785,
      "time_to_recovery": 5
    },
    {
      "condition": "ONLINE_JOINT",
      "recovered": true,
      "seed": 2,
      "target_accuracy": 0.1616,
      "time_to_recovery": 5
    },
    {
      "condition": "ONLINE_JOINT",
      "recovered": true,
      "seed": 3,
      "target_accuracy": 0.1597,
      "time_to_recovery": 5
    },
    {
      "condition": "ONLINE_JOINT",
      "recovered": true,
      "seed": 4,
      "target_accuracy": 0.1841,
      "time_to_recovery": 5
    },
    {
      "condition": "ONLINE_JOINT",
      "recovered": true,
      "seed": 5,
      "target_accuracy": 0.1935,
      "time_to_recovery": 5
    },
    {
      "condition": "ONLINE_JOINT",
      "recovered": true,
      "seed": 6,
      "target_accuracy": 0.1823,
      "time_to_recovery": 5
    },
    {
      "condition": "ONLINE_JOINT",
      "recovered": true,
      "seed": 7,
      "target_accuracy": 0.1653,
      "time_to_recovery": 5
    },
    {
      "condition": "ONLINE_JOINT",
      "recovered": true,
      "seed": 8,
      "target_accuracy": 0.1766,
      "time_to_recovery": 5
    },
    {
      "condition": "ONLINE_JOINT",
      "recovered": true,
      "seed": 9,
      "target_accuracy": 0.1672,
      "time_to_recovery": 5
    },
    {
      "condition": "ONLINE_MANAGE_ONLY",
      "recovered": true,
      "seed": 0,
      "target_accuracy": 0.1541,
      "time_to_recovery": 5
    },
    {
      "condition": "ONLINE_MANAGE_ONLY",
      "recovered": true,
      "seed": 1,
      "target_accuracy": 0.171,
      "time_to_recovery": 5
    },
    {
      "condition": "ONLINE_MANAGE_ONLY",
      "recovered": true,
      "seed": 2,
      "target_accuracy": 0.1804,
      "time_to_recovery": 5
    },
    {
      "condition": "ONLINE_MANAGE_ONLY",
      "recovered": true,
      "seed": 3,
      "target_accuracy": 0.1597,
      "time_to_recovery": 5
    },
    {
      "condition": "ONLINE_MANAGE_ONLY",
      "recovered": true,
      "seed": 4,
      "target_accuracy": 0.1559,
      "time_to_recovery": 5
    },
    {
      "condition": "ONLINE_MANAGE_ONLY",
      "recovered": true,
      "seed": 5,
      "target_accuracy": 0.201,
      "time_to_recovery": 5
    },
    {
      "condition": "ONLINE_MANAGE_ONLY",
      "recovered": true,
      "seed": 6,
      "target_accuracy": 0.1522,
      "time_to_recovery": 5
    },
    {
      "condition": "ONLINE_MANAGE_ONLY",
      "recovered": true,
      "seed": 7,
      "target_accuracy": 0.1898,
      "time_to_recovery": 5
    },
    {
      "condition": "ONLINE_MANAGE_ONLY",
      "recovered": true,
      "seed": 8,
      "target_accuracy": 0.1916,
      "time_to_recovery": 5
    },
    {
      "condition": "ONLINE_MANAGE_ONLY",
      "recovered": true,
      "seed": 9,
      "target_accuracy": 0.1672,
      "time_to_recovery": 5
    }
  ]
}