| [ |
| { |
| "baseline": "majority", |
| "ap_failure_vs_stable": 0.5, |
| "auroc_failure_vs_stable": 0.5, |
| "macro_f1_12class": 0.13333333333333333, |
| "macro_f1_used_only": 0.13333333333333333, |
| "per_class_f1_12": [ |
| 0.6666666666666666, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0 |
| ], |
| "used_classes": [ |
| 0, |
| 7, |
| 8, |
| 9, |
| 10 |
| ], |
| "fpr_at_predicted_class": 0.0, |
| "recall_at_predicted_class": 0.0, |
| "pair_rank_acc": 0.0, |
| "pair_intervention_success": 0.0, |
| "n_test": 256, |
| "n_failure": 128, |
| "n_stable": 128 |
| }, |
| { |
| "baseline": "stratified_random", |
| "ap_failure_vs_stable": 0.5100781096496319, |
| "auroc_failure_vs_stable": 0.5152587890625, |
| "macro_f1_12class": 0.25403503281753126, |
| "macro_f1_used_only": 0.25403503281753126, |
| "per_class_f1_12": [ |
| 0.5301204819277109, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.15584415584415584, |
| 0.18421052631578946, |
| 0.21818181818181817, |
| 0.18181818181818182, |
| 0.0 |
| ], |
| "used_classes": [ |
| 0, |
| 7, |
| 8, |
| 9, |
| 10 |
| ], |
| "fpr_at_predicted_class": 0.484375, |
| "recall_at_predicted_class": 0.5703125, |
| "pair_rank_acc": 0.515625, |
| "pair_intervention_success": 0.28125, |
| "n_test": 256, |
| "n_failure": 128, |
| "n_stable": 128 |
| }, |
| { |
| "baseline": "last_state", |
| "ap_failure_vs_stable": 0.5, |
| "auroc_failure_vs_stable": 0.5, |
| "macro_f1_12class": 0.08260820457215143, |
| "macro_f1_used_only": 0.08260820457215143, |
| "per_class_f1_12": [ |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.041666666666666664, |
| 0.0, |
| 0.16129032258064516, |
| 0.21008403361344538, |
| 0.0 |
| ], |
| "used_classes": [ |
| 0, |
| 7, |
| 8, |
| 9, |
| 10 |
| ], |
| "fpr_at_predicted_class": 1.0, |
| "recall_at_predicted_class": 1.0, |
| "pair_rank_acc": 0.0, |
| "pair_intervention_success": 0.0, |
| "n_test": 256, |
| "n_failure": 128, |
| "n_stable": 128 |
| }, |
| { |
| "baseline": "all_state", |
| "ap_failure_vs_stable": 0.5, |
| "auroc_failure_vs_stable": 0.5, |
| "macro_f1_12class": 0.10492109500805151, |
| "macro_f1_used_only": 0.10492109500805151, |
| "per_class_f1_12": [ |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.18, |
| 0.0, |
| 0.18518518518518517, |
| 0.15942028985507245, |
| 0.0 |
| ], |
| "used_classes": [ |
| 0, |
| 7, |
| 8, |
| 9, |
| 10 |
| ], |
| "fpr_at_predicted_class": 1.0, |
| "recall_at_predicted_class": 1.0, |
| "pair_rank_acc": 0.0, |
| "pair_intervention_success": 0.0, |
| "n_test": 256, |
| "n_failure": 128, |
| "n_stable": 128 |
| }, |
| { |
| "baseline": "all_context", |
| "ap_failure_vs_stable": 0.5, |
| "auroc_failure_vs_stable": 0.5, |
| "macro_f1_12class": 0.1667430401576743, |
| "macro_f1_used_only": 0.1667430401576743, |
| "per_class_f1_12": [ |
| 0.1111111111111111, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.24444444444444444, |
| 0.14634146341463414, |
| 0.15, |
| 0.18181818181818182, |
| 0.0 |
| ], |
| "used_classes": [ |
| 0, |
| 7, |
| 8, |
| 9, |
| 10 |
| ], |
| "fpr_at_predicted_class": 0.9375, |
| "recall_at_predicted_class": 0.9375, |
| "pair_rank_acc": 0.0, |
| "pair_intervention_success": 0.0, |
| "n_test": 256, |
| "n_failure": 128, |
| "n_stable": 128 |
| }, |
| { |
| "baseline": "all_actions", |
| "ap_failure_vs_stable": 0.5, |
| "auroc_failure_vs_stable": 0.5, |
| "macro_f1_12class": 0.11253233208176887, |
| "macro_f1_used_only": 0.11253233208176887, |
| "per_class_f1_12": [ |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.2549019607843137, |
| 0.058823529411764705, |
| 0.1, |
| 0.14893617021276595, |
| 0.0 |
| ], |
| "used_classes": [ |
| 0, |
| 7, |
| 8, |
| 9, |
| 10 |
| ], |
| "fpr_at_predicted_class": 1.0, |
| "recall_at_predicted_class": 1.0, |
| "pair_rank_acc": 0.0, |
| "pair_intervention_success": 0.0, |
| "n_test": 256, |
| "n_failure": 128, |
| "n_stable": 128 |
| }, |
| { |
| "baseline": "structured_concat", |
| "ap_failure_vs_stable": 0.5, |
| "auroc_failure_vs_stable": 0.5, |
| "macro_f1_12class": 0.08851226238550183, |
| "macro_f1_used_only": 0.08851226238550183, |
| "per_class_f1_12": [ |
| 0.015384615384615385, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.2159090909090909, |
| 0.0, |
| 0.2112676056338028, |
| 0.0 |
| ], |
| "used_classes": [ |
| 0, |
| 7, |
| 8, |
| 9, |
| 10 |
| ], |
| "fpr_at_predicted_class": 0.9921875, |
| "recall_at_predicted_class": 0.9921875, |
| "pair_rank_acc": 0.0, |
| "pair_intervention_success": 0.0, |
| "n_test": 256, |
| "n_failure": 128, |
| "n_stable": 128 |
| }, |
| { |
| "baseline": "last_frame", |
| "ap_failure_vs_stable": 0.5005103308635411, |
| "auroc_failure_vs_stable": 0.50006103515625, |
| "macro_f1_12class": 0.17176707421505463, |
| "macro_f1_used_only": 0.17176707421505463, |
| "per_class_f1_12": [ |
| 0.2967032967032967, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.21052631578947367, |
| 0.23255813953488372, |
| 0.08571428571428572, |
| 0.03333333333333333, |
| 0.0 |
| ], |
| "used_classes": [ |
| 0, |
| 7, |
| 8, |
| 9, |
| 10 |
| ], |
| "fpr_at_predicted_class": 0.7890625, |
| "recall_at_predicted_class": 0.7890625, |
| "pair_rank_acc": 0.0859375, |
| "pair_intervention_success": 0.0, |
| "n_test": 256, |
| "n_failure": 128, |
| "n_stable": 128 |
| }, |
| { |
| "baseline": "mid_frame", |
| "ap_failure_vs_stable": 0.5524097756299033, |
| "auroc_failure_vs_stable": 0.541534423828125, |
| "macro_f1_12class": 0.2442235918768214, |
| "macro_f1_used_only": 0.2442235918768214, |
| "per_class_f1_12": [ |
| 0.423963133640553, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.24719101123595505, |
| 0.2535211267605634, |
| 0.14492753623188406, |
| 0.15151515151515152, |
| 0.0 |
| ], |
| "used_classes": [ |
| 0, |
| 7, |
| 8, |
| 9, |
| 10 |
| ], |
| "fpr_at_predicted_class": 0.640625, |
| "recall_at_predicted_class": 0.6640625, |
| "pair_rank_acc": 0.2734375, |
| "pair_intervention_success": 0.0859375, |
| "n_test": 256, |
| "n_failure": 128, |
| "n_stable": 128 |
| }, |
| { |
| "baseline": "mid_frames", |
| "ap_failure_vs_stable": 0.7745800595694223, |
| "auroc_failure_vs_stable": 0.78509521484375, |
| "macro_f1_12class": 0.36406230370936254, |
| "macro_f1_used_only": 0.36406230370936254, |
| "per_class_f1_12": [ |
| 0.7214285714285714, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.37333333333333335, |
| 0.3235294117647059, |
| 0.3111111111111111, |
| 0.09090909090909091, |
| 0.0 |
| ], |
| "used_classes": [ |
| 0, |
| 7, |
| 8, |
| 9, |
| 10 |
| ], |
| "fpr_at_predicted_class": 0.2109375, |
| "recall_at_predicted_class": 0.6015625, |
| "pair_rank_acc": 0.7890625, |
| "pair_intervention_success": 0.421875, |
| "n_test": 256, |
| "n_failure": 128, |
| "n_stable": 128 |
| }, |
| { |
| "baseline": "all_frame_bag", |
| "ap_failure_vs_stable": 0.5047957454046323, |
| "auroc_failure_vs_stable": 0.5006103515625, |
| "macro_f1_12class": 0.42891805094130675, |
| "macro_f1_used_only": 0.42891805094130675, |
| "per_class_f1_12": [ |
| 0.5038759689922481, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.4523809523809524, |
| 0.4166666666666667, |
| 0.48, |
| 0.2916666666666667, |
| 0.0 |
| ], |
| "used_classes": [ |
| 0, |
| 7, |
| 8, |
| 9, |
| 10 |
| ], |
| "fpr_at_predicted_class": 0.4921875, |
| "recall_at_predicted_class": 0.4921875, |
| "pair_rank_acc": 0.578125, |
| "pair_intervention_success": 0.0, |
| "n_test": 256, |
| "n_failure": 128, |
| "n_stable": 128 |
| }, |
| { |
| "baseline": "all_frames", |
| "ap_failure_vs_stable": 0.8761761121534359, |
| "auroc_failure_vs_stable": 0.84552001953125, |
| "macro_f1_12class": 0.4896093583835519, |
| "macro_f1_used_only": 0.4896093583835519, |
| "per_class_f1_12": [ |
| 0.8111888111888111, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.4838709677419355, |
| 0.4857142857142857, |
| 0.44, |
| 0.22727272727272727, |
| 0.0 |
| ], |
| "used_classes": [ |
| 0, |
| 7, |
| 8, |
| 9, |
| 10 |
| ], |
| "fpr_at_predicted_class": 0.09375, |
| "recall_at_predicted_class": 0.671875, |
| "pair_rank_acc": 0.8828125, |
| "pair_intervention_success": 0.6015625, |
| "n_test": 256, |
| "n_failure": 128, |
| "n_stable": 128 |
| }, |
| { |
| "baseline": "frame_deltas", |
| "ap_failure_vs_stable": 0.8525414381864486, |
| "auroc_failure_vs_stable": 0.83453369140625, |
| "macro_f1_12class": 0.391703872303612, |
| "macro_f1_used_only": 0.391703872303612, |
| "per_class_f1_12": [ |
| 0.7794117647058824, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.4666666666666667, |
| 0.3582089552238806, |
| 0.18181818181818182, |
| 0.1724137931034483, |
| 0.0 |
| ], |
| "used_classes": [ |
| 0, |
| 7, |
| 8, |
| 9, |
| 10 |
| ], |
| "fpr_at_predicted_class": 0.171875, |
| "recall_at_predicted_class": 0.703125, |
| "pair_rank_acc": 0.8515625, |
| "pair_intervention_success": 0.5625, |
| "n_test": 256, |
| "n_failure": 128, |
| "n_stable": 128 |
| }, |
| { |
| "baseline": "frame_delta_summary", |
| "ap_failure_vs_stable": 0.7521299066055007, |
| "auroc_failure_vs_stable": 0.7440185546875, |
| "macro_f1_12class": 0.46544424443256965, |
| "macro_f1_used_only": 0.46544424443256965, |
| "per_class_f1_12": [ |
| 0.7203065134099617, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.5373134328358209, |
| 0.36363636363636365, |
| 0.32, |
| 0.38596491228070173, |
| 0.0 |
| ], |
| "used_classes": [ |
| 0, |
| 7, |
| 8, |
| 9, |
| 10 |
| ], |
| "fpr_at_predicted_class": 0.265625, |
| "recall_at_predicted_class": 0.6953125, |
| "pair_rank_acc": 0.875, |
| "pair_intervention_success": 0.4453125, |
| "n_test": 256, |
| "n_failure": 128, |
| "n_stable": 128 |
| }, |
| { |
| "baseline": "mid_frame_deltas", |
| "ap_failure_vs_stable": 0.7192851926384367, |
| "auroc_failure_vs_stable": 0.707000732421875, |
| "macro_f1_12class": 0.27040067183597294, |
| "macro_f1_used_only": 0.27040067183597294, |
| "per_class_f1_12": [ |
| 0.6641221374045801, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.23880597014925373, |
| 0.1917808219178082, |
| 0.15384615384615385, |
| 0.10344827586206896, |
| 0.0 |
| ], |
| "used_classes": [ |
| 0, |
| 7, |
| 8, |
| 9, |
| 10 |
| ], |
| "fpr_at_predicted_class": 0.3203125, |
| "recall_at_predicted_class": 0.6328125, |
| "pair_rank_acc": 0.6875, |
| "pair_intervention_success": 0.390625, |
| "n_test": 256, |
| "n_failure": 128, |
| "n_stable": 128 |
| }, |
| { |
| "baseline": "mid_frame_max_abs", |
| "ap_failure_vs_stable": 0.6227656163283489, |
| "auroc_failure_vs_stable": 0.61810302734375, |
| "macro_f1_12class": 0.27295446445967725, |
| "macro_f1_used_only": 0.27295446445967725, |
| "per_class_f1_12": [ |
| 0.568, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.2716049382716049, |
| 0.36585365853658536, |
| 0.041666666666666664, |
| 0.11764705882352941, |
| 0.0 |
| ], |
| "used_classes": [ |
| 0, |
| 7, |
| 8, |
| 9, |
| 10 |
| ], |
| "fpr_at_predicted_class": 0.4453125, |
| "recall_at_predicted_class": 0.6015625, |
| "pair_rank_acc": 0.6640625, |
| "pair_intervention_success": 0.21875, |
| "n_test": 256, |
| "n_failure": 128, |
| "n_stable": 128 |
| }, |
| { |
| "baseline": "simple_temporal_mlp_frame_delta_summary", |
| "ap_failure_vs_stable": 0.5473804380398812, |
| "auroc_failure_vs_stable": 0.542724609375, |
| "macro_f1_12class": 0.21759726107552196, |
| "macro_f1_used_only": 0.21759726107552196, |
| "per_class_f1_12": [ |
| 0.4980237154150198, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.13186813186813187, |
| 0.18181818181818182, |
| 0.2222222222222222, |
| 0.05405405405405406, |
| 0.0 |
| ], |
| "used_classes": [ |
| 0, |
| 7, |
| 8, |
| 9, |
| 10 |
| ], |
| "fpr_at_predicted_class": 0.5078125, |
| "recall_at_predicted_class": 0.515625, |
| "pair_rank_acc": 0.7109375, |
| "pair_intervention_success": 0.0625, |
| "n_test": 256, |
| "n_failure": 128, |
| "n_stable": 128 |
| }, |
| { |
| "baseline": "pair_oracle_safe_template", |
| "ap_failure_vs_stable": 1.0, |
| "auroc_failure_vs_stable": 1.0, |
| "macro_f1_12class": 1.0, |
| "macro_f1_used_only": 1.0, |
| "per_class_f1_12": [ |
| 1.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 1.0, |
| 1.0, |
| 1.0, |
| 1.0, |
| 0.0 |
| ], |
| "used_classes": [ |
| 0, |
| 7, |
| 8, |
| 9, |
| 10 |
| ], |
| "fpr_at_predicted_class": 0.0, |
| "recall_at_predicted_class": 1.0, |
| "pair_rank_acc": 1.0, |
| "pair_intervention_success": 1.0, |
| "n_test": 256, |
| "n_failure": 128, |
| "n_stable": 128 |
| } |
| ] |
