[
  {
    "baseline": "majority",
    "ap_failure_vs_stable": 0.5,
    "auroc_failure_vs_stable": 0.5,
    "macro_f1_12class": 0.13333333333333333,
    "macro_f1_used_only": 0.13333333333333333,
    "per_class_f1_12": [
      0.6666666666666666,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0
    ],
    "used_classes": [
      0,
      7,
      8,
      9,
      10
    ],
    "fpr_at_predicted_class": 0.0,
    "recall_at_predicted_class": 0.0,
    "pair_rank_acc": 0.0,
    "pair_intervention_success": 0.0,
    "n_test": 256,
    "n_failure": 128,
    "n_stable": 128
  },
  {
    "baseline": "stratified_random",
    "ap_failure_vs_stable": 0.5100781096496319,
    "auroc_failure_vs_stable": 0.5152587890625,
    "macro_f1_12class": 0.25403503281753126,
    "macro_f1_used_only": 0.25403503281753126,
    "per_class_f1_12": [
      0.5301204819277109,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.15584415584415584,
      0.18421052631578946,
      0.21818181818181817,
      0.18181818181818182,
      0.0
    ],
    "used_classes": [
      0,
      7,
      8,
      9,
      10
    ],
    "fpr_at_predicted_class": 0.484375,
    "recall_at_predicted_class": 0.5703125,
    "pair_rank_acc": 0.515625,
    "pair_intervention_success": 0.28125,
    "n_test": 256,
    "n_failure": 128,
    "n_stable": 128
  },
  {
    "baseline": "last_state",
    "ap_failure_vs_stable": 0.5,
    "auroc_failure_vs_stable": 0.5,
    "macro_f1_12class": 0.08260820457215143,
    "macro_f1_used_only": 0.08260820457215143,
    "per_class_f1_12": [
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.041666666666666664,
      0.0,
      0.16129032258064516,
      0.21008403361344538,
      0.0
    ],
    "used_classes": [
      0,
      7,
      8,
      9,
      10
    ],
    "fpr_at_predicted_class": 1.0,
    "recall_at_predicted_class": 1.0,
    "pair_rank_acc": 0.0,
    "pair_intervention_success": 0.0,
    "n_test": 256,
    "n_failure": 128,
    "n_stable": 128
  },
  {
    "baseline": "all_state",
    "ap_failure_vs_stable": 0.5,
    "auroc_failure_vs_stable": 0.5,
    "macro_f1_12class": 0.10492109500805151,
    "macro_f1_used_only": 0.10492109500805151,
    "per_class_f1_12": [
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.18,
      0.0,
      0.18518518518518517,
      0.15942028985507245,
      0.0
    ],
    "used_classes": [
      0,
      7,
      8,
      9,
      10
    ],
    "fpr_at_predicted_class": 1.0,
    "recall_at_predicted_class": 1.0,
    "pair_rank_acc": 0.0,
    "pair_intervention_success": 0.0,
    "n_test": 256,
    "n_failure": 128,
    "n_stable": 128
  },
  {
    "baseline": "all_context",
    "ap_failure_vs_stable": 0.5,
    "auroc_failure_vs_stable": 0.5,
    "macro_f1_12class": 0.1667430401576743,
    "macro_f1_used_only": 0.1667430401576743,
    "per_class_f1_12": [
      0.1111111111111111,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.24444444444444444,
      0.14634146341463414,
      0.15,
      0.18181818181818182,
      0.0
    ],
    "used_classes": [
      0,
      7,
      8,
      9,
      10
    ],
    "fpr_at_predicted_class": 0.9375,
    "recall_at_predicted_class": 0.9375,
    "pair_rank_acc": 0.0,
    "pair_intervention_success": 0.0,
    "n_test": 256,
    "n_failure": 128,
    "n_stable": 128
  },
  {
    "baseline": "all_actions",
    "ap_failure_vs_stable": 0.5,
    "auroc_failure_vs_stable": 0.5,
    "macro_f1_12class": 0.11253233208176887,
    "macro_f1_used_only": 0.11253233208176887,
    "per_class_f1_12": [
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.2549019607843137,
      0.058823529411764705,
      0.1,
      0.14893617021276595,
      0.0
    ],
    "used_classes": [
      0,
      7,
      8,
      9,
      10
    ],
    "fpr_at_predicted_class": 1.0,
    "recall_at_predicted_class": 1.0,
    "pair_rank_acc": 0.0,
    "pair_intervention_success": 0.0,
    "n_test": 256,
    "n_failure": 128,
    "n_stable": 128
  },
  {
    "baseline": "structured_concat",
    "ap_failure_vs_stable": 0.5,
    "auroc_failure_vs_stable": 0.5,
    "macro_f1_12class": 0.08851226238550183,
    "macro_f1_used_only": 0.08851226238550183,
    "per_class_f1_12": [
      0.015384615384615385,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.2159090909090909,
      0.0,
      0.2112676056338028,
      0.0
    ],
    "used_classes": [
      0,
      7,
      8,
      9,
      10
    ],
    "fpr_at_predicted_class": 0.9921875,
    "recall_at_predicted_class": 0.9921875,
    "pair_rank_acc": 0.0,
    "pair_intervention_success": 0.0,
    "n_test": 256,
    "n_failure": 128,
    "n_stable": 128
  },
  {
    "baseline": "last_frame",
    "ap_failure_vs_stable": 0.5005103308635411,
    "auroc_failure_vs_stable": 0.50006103515625,
    "macro_f1_12class": 0.17176707421505463,
    "macro_f1_used_only": 0.17176707421505463,
    "per_class_f1_12": [
      0.2967032967032967,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.21052631578947367,
      0.23255813953488372,
      0.08571428571428572,
      0.03333333333333333,
      0.0
    ],
    "used_classes": [
      0,
      7,
      8,
      9,
      10
    ],
    "fpr_at_predicted_class": 0.7890625,
    "recall_at_predicted_class": 0.7890625,
    "pair_rank_acc": 0.0859375,
    "pair_intervention_success": 0.0,
    "n_test": 256,
    "n_failure": 128,
    "n_stable": 128
  },
  {
    "baseline": "mid_frame",
    "ap_failure_vs_stable": 0.5524097756299033,
    "auroc_failure_vs_stable": 0.541534423828125,
    "macro_f1_12class": 0.2442235918768214,
    "macro_f1_used_only": 0.2442235918768214,
    "per_class_f1_12": [
      0.423963133640553,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.24719101123595505,
      0.2535211267605634,
      0.14492753623188406,
      0.15151515151515152,
      0.0
    ],
    "used_classes": [
      0,
      7,
      8,
      9,
      10
    ],
    "fpr_at_predicted_class": 0.640625,
    "recall_at_predicted_class": 0.6640625,
    "pair_rank_acc": 0.2734375,
    "pair_intervention_success": 0.0859375,
    "n_test": 256,
    "n_failure": 128,
    "n_stable": 128
  },
  {
    "baseline": "mid_frames",
    "ap_failure_vs_stable": 0.7745800595694223,
    "auroc_failure_vs_stable": 0.78509521484375,
    "macro_f1_12class": 0.36406230370936254,
    "macro_f1_used_only": 0.36406230370936254,
    "per_class_f1_12": [
      0.7214285714285714,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.37333333333333335,
      0.3235294117647059,
      0.3111111111111111,
      0.09090909090909091,
      0.0
    ],
    "used_classes": [
      0,
      7,
      8,
      9,
      10
    ],
    "fpr_at_predicted_class": 0.2109375,
    "recall_at_predicted_class": 0.6015625,
    "pair_rank_acc": 0.7890625,
    "pair_intervention_success": 0.421875,
    "n_test": 256,
    "n_failure": 128,
    "n_stable": 128
  },
  {
    "baseline": "all_frame_bag",
    "ap_failure_vs_stable": 0.5047957454046323,
    "auroc_failure_vs_stable": 0.5006103515625,
    "macro_f1_12class": 0.42891805094130675,
    "macro_f1_used_only": 0.42891805094130675,
    "per_class_f1_12": [
      0.5038759689922481,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.4523809523809524,
      0.4166666666666667,
      0.48,
      0.2916666666666667,
      0.0
    ],
    "used_classes": [
      0,
      7,
      8,
      9,
      10
    ],
    "fpr_at_predicted_class": 0.4921875,
    "recall_at_predicted_class": 0.4921875,
    "pair_rank_acc": 0.578125,
    "pair_intervention_success": 0.0,
    "n_test": 256,
    "n_failure": 128,
    "n_stable": 128
  },
  {
    "baseline": "all_frames",
    "ap_failure_vs_stable": 0.8761761121534359,
    "auroc_failure_vs_stable": 0.84552001953125,
    "macro_f1_12class": 0.4896093583835519,
    "macro_f1_used_only": 0.4896093583835519,
    "per_class_f1_12": [
      0.8111888111888111,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.4838709677419355,
      0.4857142857142857,
      0.44,
      0.22727272727272727,
      0.0
    ],
    "used_classes": [
      0,
      7,
      8,
      9,
      10
    ],
    "fpr_at_predicted_class": 0.09375,
    "recall_at_predicted_class": 0.671875,
    "pair_rank_acc": 0.8828125,
    "pair_intervention_success": 0.6015625,
    "n_test": 256,
    "n_failure": 128,
    "n_stable": 128
  },
  {
    "baseline": "frame_deltas",
    "ap_failure_vs_stable": 0.8525414381864486,
    "auroc_failure_vs_stable": 0.83453369140625,
    "macro_f1_12class": 0.391703872303612,
    "macro_f1_used_only": 0.391703872303612,
    "per_class_f1_12": [
      0.7794117647058824,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.4666666666666667,
      0.3582089552238806,
      0.18181818181818182,
      0.1724137931034483,
      0.0
    ],
    "used_classes": [
      0,
      7,
      8,
      9,
      10
    ],
    "fpr_at_predicted_class": 0.171875,
    "recall_at_predicted_class": 0.703125,
    "pair_rank_acc": 0.8515625,
    "pair_intervention_success": 0.5625,
    "n_test": 256,
    "n_failure": 128,
    "n_stable": 128
  },
  {
    "baseline": "frame_delta_summary",
    "ap_failure_vs_stable": 0.7521299066055007,
    "auroc_failure_vs_stable": 0.7440185546875,
    "macro_f1_12class": 0.46544424443256965,
    "macro_f1_used_only": 0.46544424443256965,
    "per_class_f1_12": [
      0.7203065134099617,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.5373134328358209,
      0.36363636363636365,
      0.32,
      0.38596491228070173,
      0.0
    ],
    "used_classes": [
      0,
      7,
      8,
      9,
      10
    ],
    "fpr_at_predicted_class": 0.265625,
    "recall_at_predicted_class": 0.6953125,
    "pair_rank_acc": 0.875,
    "pair_intervention_success": 0.4453125,
    "n_test": 256,
    "n_failure": 128,
    "n_stable": 128
  },
  {
    "baseline": "mid_frame_deltas",
    "ap_failure_vs_stable": 0.7192851926384367,
    "auroc_failure_vs_stable": 0.707000732421875,
    "macro_f1_12class": 0.27040067183597294,
    "macro_f1_used_only": 0.27040067183597294,
    "per_class_f1_12": [
      0.6641221374045801,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.23880597014925373,
      0.1917808219178082,
      0.15384615384615385,
      0.10344827586206896,
      0.0
    ],
    "used_classes": [
      0,
      7,
      8,
      9,
      10
    ],
    "fpr_at_predicted_class": 0.3203125,
    "recall_at_predicted_class": 0.6328125,
    "pair_rank_acc": 0.6875,
    "pair_intervention_success": 0.390625,
    "n_test": 256,
    "n_failure": 128,
    "n_stable": 128
  },
  {
    "baseline": "mid_frame_max_abs",
    "ap_failure_vs_stable": 0.6227656163283489,
    "auroc_failure_vs_stable": 0.61810302734375,
    "macro_f1_12class": 0.27295446445967725,
    "macro_f1_used_only": 0.27295446445967725,
    "per_class_f1_12": [
      0.568,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.2716049382716049,
      0.36585365853658536,
      0.041666666666666664,
      0.11764705882352941,
      0.0
    ],
    "used_classes": [
      0,
      7,
      8,
      9,
      10
    ],
    "fpr_at_predicted_class": 0.4453125,
    "recall_at_predicted_class": 0.6015625,
    "pair_rank_acc": 0.6640625,
    "pair_intervention_success": 0.21875,
    "n_test": 256,
    "n_failure": 128,
    "n_stable": 128
  },
  {
    "baseline": "simple_temporal_mlp_frame_delta_summary",
    "ap_failure_vs_stable": 0.5473804380398812,
    "auroc_failure_vs_stable": 0.542724609375,
    "macro_f1_12class": 0.21759726107552196,
    "macro_f1_used_only": 0.21759726107552196,
    "per_class_f1_12": [
      0.4980237154150198,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.13186813186813187,
      0.18181818181818182,
      0.2222222222222222,
      0.05405405405405406,
      0.0
    ],
    "used_classes": [
      0,
      7,
      8,
      9,
      10
    ],
    "fpr_at_predicted_class": 0.5078125,
    "recall_at_predicted_class": 0.515625,
    "pair_rank_acc": 0.7109375,
    "pair_intervention_success": 0.0625,
    "n_test": 256,
    "n_failure": 128,
    "n_stable": 128
  },
  {
    "baseline": "pair_oracle_safe_template",
    "ap_failure_vs_stable": 1.0,
    "auroc_failure_vs_stable": 1.0,
    "macro_f1_12class": 1.0,
    "macro_f1_used_only": 1.0,
    "per_class_f1_12": [
      1.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      0.0,
      1.0,
      1.0,
      1.0,
      1.0,
      0.0
    ],
    "used_classes": [
      0,
      7,
      8,
      9,
      10
    ],
    "fpr_at_predicted_class": 0.0,
    "recall_at_predicted_class": 1.0,
    "pair_rank_acc": 1.0,
    "pair_intervention_success": 1.0,
    "n_test": 256,
    "n_failure": 128,
    "n_stable": 128
  }
]