{ "provenance": { "generator": "verify.py", "pool": "VAL5000", "split": "val2017", "n_images": 5000, "selection": "all 5000 val2017 images, used only to report", "task": "image-level category presence (binary, one rule per category)", "protocol": "pairs chosen on train2017, scored here on a class-balanced subsample so chance is 0.5 for every category" }, "rules": { "person": { "cat_id": 1, "pos_dim": 48, "neg_dim": 637, "n_balanced": 4614, "train_balanced_accuracy": 0.8612, "balanced_accuracy": 0.8552, "F1": 0.8565, "precision": 0.8492, "recall": 0.8639 }, "bicycle": { "cat_id": 2, "pos_dim": 658, "neg_dim": 378, "n_balanced": 298, "train_balanced_accuracy": 0.7708, "balanced_accuracy": 0.7651, "F1": 0.7426, "precision": 0.8211, "recall": 0.6779 }, "car": { "cat_id": 3, "pos_dim": 578, "neg_dim": 257, "n_balanced": 1070, "train_balanced_accuracy": 0.7412, "balanced_accuracy": 0.7561, "F1": 0.7603, "precision": 0.7473, "recall": 0.7738 }, "motorcycle": { "cat_id": 4, "pos_dim": 348, "neg_dim": 239, "n_balanced": 318, "train_balanced_accuracy": 0.8245, "balanced_accuracy": 0.8145, "F1": 0.8338, "precision": 0.7551, "recall": 0.9308 }, "airplane": { "cat_id": 5, "pos_dim": 301, "neg_dim": 96, "n_balanced": 194, "train_balanced_accuracy": 0.9128, "balanced_accuracy": 0.8969, "F1": 0.8936, "precision": 0.9231, "recall": 0.866 }, "bus": { "cat_id": 6, "pos_dim": 11, "neg_dim": 642, "n_balanced": 378, "train_balanced_accuracy": 0.825, "balanced_accuracy": 0.8148, "F1": 0.8301, "precision": 0.7668, "recall": 0.9048 }, "train": { "cat_id": 7, "pos_dim": 460, "neg_dim": 378, "n_balanced": 314, "train_balanced_accuracy": 0.9082, "balanced_accuracy": 0.9268, "F1": 0.9256, "precision": 0.9408, "recall": 0.9108 }, "truck": { "cat_id": 8, "pos_dim": 578, "neg_dim": 642, "n_balanced": 500, "train_balanced_accuracy": 0.7486, "balanced_accuracy": 0.738, "F1": 0.7596, "precision": 0.7017, "recall": 0.828 }, "boat": { "cat_id": 9, "pos_dim": 481, "neg_dim": 642, "n_balanced": 242, "train_balanced_accuracy": 0.783, "balanced_accuracy": 0.8058, "F1": 0.8185, "precision": 0.7681, "recall": 0.876 }, "traffic light": { "cat_id": 10, "pos_dim": 578, "neg_dim": 262, "n_balanced": 382, "train_balanced_accuracy": 0.8318, "balanced_accuracy": 0.8194, "F1": 0.8321, "precision": 0.7773, "recall": 0.8953 }, "fire hydrant": { "cat_id": 11, "pos_dim": 656, "neg_dim": 262, "n_balanced": 172, "train_balanced_accuracy": 0.822, "balanced_accuracy": 0.8605, "F1": 0.8571, "precision": 0.878, "recall": 0.8372 }, "stop sign": { "cat_id": 13, "pos_dim": 544, "neg_dim": 378, "n_balanced": 138, "train_balanced_accuracy": 0.8183, "balanced_accuracy": 0.8188, "F1": 0.8148, "precision": 0.8333, "recall": 0.7971 }, "parking meter": { "cat_id": 14, "pos_dim": 739, "neg_dim": 213, "n_balanced": 74, "train_balanced_accuracy": 0.8355, "balanced_accuracy": 0.8243, "F1": 0.7937, "precision": 0.9615, "recall": 0.6757 }, "bench": { "cat_id": 15, "pos_dim": 710, "neg_dim": 617, "n_balanced": 470, "train_balanced_accuracy": 0.6825, "balanced_accuracy": 0.6702, "F1": 0.6764, "precision": 0.6639, "recall": 0.6894 }, "bird": { "cat_id": 16, "pos_dim": 82, "neg_dim": 48, "n_balanced": 250, "train_balanced_accuracy": 0.7661, "balanced_accuracy": 0.768, "F1": 0.7456, "precision": 0.8252, "recall": 0.68 }, "cat": { "cat_id": 17, "pos_dim": 82, "neg_dim": 240, "n_balanced": 368, "train_balanced_accuracy": 0.8839, "balanced_accuracy": 0.8587, "F1": 0.8667, "precision": 0.8204, "recall": 0.9185 }, "dog": { "cat_id": 18, "pos_dim": 294, "neg_dim": 25, "n_balanced": 354, "train_balanced_accuracy": 0.809, "balanced_accuracy": 0.8249, "F1": 0.836, "precision": 0.7861, "recall": 0.8927 }, "horse": { "cat_id": 19, "pos_dim": 365, "neg_dim": 35, "n_balanced": 256, "train_balanced_accuracy": 0.8354, "balanced_accuracy": 0.8281, "F1": 0.8394, "precision": 0.7877, "recall": 0.8984 }, "sheep": { "cat_id": 20, "pos_dim": 204, "neg_dim": 592, "n_balanced": 130, "train_balanced_accuracy": 0.8852, "balanced_accuracy": 0.9077, "F1": 0.9118, "precision": 0.8732, "recall": 0.9538 }, "cow": { "cat_id": 21, "pos_dim": 222, "neg_dim": 567, "n_balanced": 174, "train_balanced_accuracy": 0.8895, "balanced_accuracy": 0.8621, "F1": 0.871, "precision": 0.8182, "recall": 0.931 }, "elephant": { "cat_id": 22, "pos_dim": 523, "neg_dim": 713, "n_balanced": 178, "train_balanced_accuracy": 0.9188, "balanced_accuracy": 0.9326, "F1": 0.9348, "precision": 0.9053, "recall": 0.9663 }, "bear": { "cat_id": 23, "pos_dim": 49, "neg_dim": 297, "n_balanced": 98, "train_balanced_accuracy": 0.9036, "balanced_accuracy": 0.8673, "F1": 0.8713, "precision": 0.8462, "recall": 0.898 }, "zebra": { "cat_id": 24, "pos_dim": 23, "neg_dim": 380, "n_balanced": 170, "train_balanced_accuracy": 0.9329, "balanced_accuracy": 0.9176, "F1": 0.9213, "precision": 0.8817, "recall": 0.9647 }, "giraffe": { "cat_id": 25, "pos_dim": 436, "neg_dim": 713, "n_balanced": 202, "train_balanced_accuracy": 0.9535, "balanced_accuracy": 0.9653, "F1": 0.9665, "precision": 0.9352, "recall": 1.0 }, "backpack": { "cat_id": 27, "pos_dim": 48, "neg_dim": 520, "n_balanced": 456, "train_balanced_accuracy": 0.6951, "balanced_accuracy": 0.6732, "F1": 0.747, "precision": 0.6094, "recall": 0.9649 }, "umbrella": { "cat_id": 28, "pos_dim": 670, "neg_dim": 650, "n_balanced": 348, "train_balanced_accuracy": 0.7821, "balanced_accuracy": 0.7672, "F1": 0.7918, "precision": 0.7163, "recall": 0.8851 }, "handbag": { "cat_id": 31, "pos_dim": 48, "neg_dim": 520, "n_balanced": 584, "train_balanced_accuracy": 0.6978, "balanced_accuracy": 0.714, "F1": 0.769, "precision": 0.645, "recall": 0.9521 }, "tie": { "cat_id": 32, "pos_dim": 552, "neg_dim": 583, "n_balanced": 290, "train_balanced_accuracy": 0.752, "balanced_accuracy": 0.7621, "F1": 0.7767, "precision": 0.7317, "recall": 0.8276 }, "suitcase": { "cat_id": 33, "pos_dim": 592, "neg_dim": 614, "n_balanced": 210, "train_balanced_accuracy": 0.8343, "balanced_accuracy": 0.8429, "F1": 0.8358, "precision": 0.875, "recall": 0.8 }, "frisbee": { "cat_id": 34, "pos_dim": 467, "neg_dim": 617, "n_balanced": 168, "train_balanced_accuracy": 0.8141, "balanced_accuracy": 0.8333, "F1": 0.8353, "precision": 0.8256, "recall": 0.8452 }, "skis": { "cat_id": 35, "pos_dim": 678, "neg_dim": 659, "n_balanced": 240, "train_balanced_accuracy": 0.9119, "balanced_accuracy": 0.8792, "F1": 0.8863, "precision": 0.837, "recall": 0.9417 }, "snowboard": { "cat_id": 36, "pos_dim": 111, "neg_dim": 659, "n_balanced": 98, "train_balanced_accuracy": 0.9011, "balanced_accuracy": 0.9388, "F1": 0.94, "precision": 0.9216, "recall": 0.9592 }, "sports ball": { "cat_id": 37, "pos_dim": 724, "neg_dim": 252, "n_balanced": 338, "train_balanced_accuracy": 0.864, "balanced_accuracy": 0.8166, "F1": 0.8306, "precision": 0.7716, "recall": 0.8994 }, "kite": { "cat_id": 38, "pos_dim": 63, "neg_dim": 330, "n_balanced": 182, "train_balanced_accuracy": 0.8576, "balanced_accuracy": 0.8791, "F1": 0.8778, "precision": 0.8876, "recall": 0.8681 }, "baseball bat": { "cat_id": 39, "pos_dim": 755, "neg_dim": 214, "n_balanced": 194, "train_balanced_accuracy": 0.9268, "balanced_accuracy": 0.9175, "F1": 0.9175, "precision": 0.9175, "recall": 0.9175 }, "baseball glove": { "cat_id": 40, "pos_dim": 349, "neg_dim": 46, "n_balanced": 200, "train_balanced_accuracy": 0.9245, "balanced_accuracy": 0.935, "F1": 0.9378, "precision": 0.8991, "recall": 0.98 }, "skateboard": { "cat_id": 41, "pos_dim": 681, "neg_dim": 378, "n_balanced": 254, "train_balanced_accuracy": 0.9025, "balanced_accuracy": 0.9055, "F1": 0.9104, "precision": 0.8652, "recall": 0.9606 }, "surfboard": { "cat_id": 42, "pos_dim": 184, "neg_dim": 647, "n_balanced": 298, "train_balanced_accuracy": 0.8746, "balanced_accuracy": 0.9195, "F1": 0.9211, "precision": 0.9032, "recall": 0.9396 }, "tennis racket": { "cat_id": 43, "pos_dim": 275, "neg_dim": 297, "n_balanced": 334, "train_balanced_accuracy": 0.9203, "balanced_accuracy": 0.9222, "F1": 0.9244, "precision": 0.8983, "recall": 0.9521 }, "bottle": { "cat_id": 44, "pos_dim": 380, "neg_dim": 240, "n_balanced": 758, "train_balanced_accuracy": 0.7426, "balanced_accuracy": 0.7256, "F1": 0.7457, "precision": 0.6948, "recall": 0.8047 }, "wine glass": { "cat_id": 46, "pos_dim": 304, "neg_dim": 8, "n_balanced": 220, "train_balanced_accuracy": 0.7821, "balanced_accuracy": 0.7682, "F1": 0.7884, "precision": 0.7252, "recall": 0.8636 }, "cup": { "cat_id": 47, "pos_dim": 642, "neg_dim": 548, "n_balanced": 780, "train_balanced_accuracy": 0.7585, "balanced_accuracy": 0.7462, "F1": 0.7815, "precision": 0.686, "recall": 0.9077 }, "fork": { "cat_id": 48, "pos_dim": 565, "neg_dim": 338, "n_balanced": 310, "train_balanced_accuracy": 0.8079, "balanced_accuracy": 0.8032, "F1": 0.8211, "precision": 0.7527, "recall": 0.9032 }, "knife": { "cat_id": 49, "pos_dim": 528, "neg_dim": 338, "n_balanced": 362, "train_balanced_accuracy": 0.795, "balanced_accuracy": 0.7707, "F1": 0.8029, "precision": 0.7042, "recall": 0.9337 }, "spoon": { "cat_id": 50, "pos_dim": 466, "neg_dim": 338, "n_balanced": 306, "train_balanced_accuracy": 0.8007, "balanced_accuracy": 0.7908, "F1": 0.8072, "precision": 0.7486, "recall": 0.8758 }, "bowl": { "cat_id": 51, "pos_dim": 490, "neg_dim": 8, "n_balanced": 628, "train_balanced_accuracy": 0.786, "balanced_accuracy": 0.7946, "F1": 0.8077, "precision": 0.7591, "recall": 0.8631 }, "banana": { "cat_id": 52, "pos_dim": 356, "neg_dim": 8, "n_balanced": 206, "train_balanced_accuracy": 0.8455, "balanced_accuracy": 0.8689, "F1": 0.88, "precision": 0.8115, "recall": 0.9612 }, "apple": { "cat_id": 53, "pos_dim": 626, "neg_dim": 583, "n_balanced": 152, "train_balanced_accuracy": 0.8102, "balanced_accuracy": 0.7632, "F1": 0.7722, "precision": 0.7439, "recall": 0.8026 }, "sandwich": { "cat_id": 54, "pos_dim": 228, "neg_dim": 258, "n_balanced": 196, "train_balanced_accuracy": 0.8501, "balanced_accuracy": 0.8061, "F1": 0.8241, "precision": 0.7542, "recall": 0.9082 }, "orange": { "cat_id": 55, "pos_dim": 626, "neg_dim": 8, "n_balanced": 170, "train_balanced_accuracy": 0.8264, "balanced_accuracy": 0.8529, "F1": 0.8503, "precision": 0.8659, "recall": 0.8353 }, "broccoli": { "cat_id": 56, "pos_dim": 156, "neg_dim": 240, "n_balanced": 142, "train_balanced_accuracy": 0.8886, "balanced_accuracy": 0.8803, "F1": 0.8811, "precision": 0.875, "recall": 0.8873 }, "carrot": { "cat_id": 57, "pos_dim": 92, "neg_dim": 8, "n_balanced": 162, "train_balanced_accuracy": 0.8568, "balanced_accuracy": 0.8457, "F1": 0.8521, "precision": 0.8182, "recall": 0.8889 }, "hot dog": { "cat_id": 58, "pos_dim": 228, "neg_dim": 258, "n_balanced": 102, "train_balanced_accuracy": 0.8584, "balanced_accuracy": 0.8529, "F1": 0.8649, "precision": 0.8, "recall": 0.9412 }, "pizza": { "cat_id": 59, "pos_dim": 321, "neg_dim": 729, "n_balanced": 306, "train_balanced_accuracy": 0.8688, "balanced_accuracy": 0.8497, "F1": 0.8571, "precision": 0.8166, "recall": 0.902 }, "donut": { "cat_id": 60, "pos_dim": 66, "neg_dim": 258, "n_balanced": 124, "train_balanced_accuracy": 0.8503, "balanced_accuracy": 0.8145, "F1": 0.816, "precision": 0.8095, "recall": 0.8226 }, "cake": { "cat_id": 61, "pos_dim": 466, "neg_dim": 338, "n_balanced": 248, "train_balanced_accuracy": 0.794, "balanced_accuracy": 0.7863, "F1": 0.803, "precision": 0.7448, "recall": 0.871 }, "chair": { "cat_id": 62, "pos_dim": 380, "neg_dim": 240, "n_balanced": 1160, "train_balanced_accuracy": 0.6913, "balanced_accuracy": 0.6931, "F1": 0.6952, "precision": 0.6905, "recall": 0.7 }, "couch": { "cat_id": 63, "pos_dim": 293, "neg_dim": 755, "n_balanced": 390, "train_balanced_accuracy": 0.7939, "balanced_accuracy": 0.7718, "F1": 0.7735, "precision": 0.7677, "recall": 0.7795 }, "potted plant": { "cat_id": 64, "pos_dim": 613, "neg_dim": 8, "n_balanced": 344, "train_balanced_accuracy": 0.6988, "balanced_accuracy": 0.7151, "F1": 0.7394, "precision": 0.6814, "recall": 0.8081 }, "bed": { "cat_id": 65, "pos_dim": 7, "neg_dim": 240, "n_balanced": 298, "train_balanced_accuracy": 0.8631, "balanced_accuracy": 0.8356, "F1": 0.8244, "precision": 0.8846, "recall": 0.7718 }, "dining table": { "cat_id": 67, "pos_dim": 565, "neg_dim": 31, "n_balanced": 1002, "train_balanced_accuracy": 0.7556, "balanced_accuracy": 0.7595, "F1": 0.7694, "precision": 0.739, "recall": 0.8024 }, "toilet": { "cat_id": 70, "pos_dim": 105, "neg_dim": 240, "n_balanced": 298, "train_balanced_accuracy": 0.9463, "balanced_accuracy": 0.943, "F1": 0.9431, "precision": 0.94, "recall": 0.9463 }, "tv": { "cat_id": 72, "pos_dim": 265, "neg_dim": 548, "n_balanced": 414, "train_balanced_accuracy": 0.8002, "balanced_accuracy": 0.7754, "F1": 0.7365, "precision": 0.8904, "recall": 0.628 }, "laptop": { "cat_id": 73, "pos_dim": 720, "neg_dim": 548, "n_balanced": 366, "train_balanced_accuracy": 0.837, "balanced_accuracy": 0.8497, "F1": 0.8525, "precision": 0.8368, "recall": 0.8689 }, "mouse": { "cat_id": 74, "pos_dim": 639, "neg_dim": 548, "n_balanced": 176, "train_balanced_accuracy": 0.907, "balanced_accuracy": 0.8693, "F1": 0.8715, "precision": 0.8571, "recall": 0.8864 }, "remote": { "cat_id": 75, "pos_dim": 642, "neg_dim": 240, "n_balanced": 290, "train_balanced_accuracy": 0.7877, "balanced_accuracy": 0.7517, "F1": 0.7844, "precision": 0.6931, "recall": 0.9034 }, "keyboard": { "cat_id": 76, "pos_dim": 639, "neg_dim": 548, "n_balanced": 212, "train_balanced_accuracy": 0.8735, "balanced_accuracy": 0.8396, "F1": 0.8365, "precision": 0.8529, "recall": 0.8208 }, "cell phone": { "cat_id": 77, "pos_dim": 592, "neg_dim": 548, "n_balanced": 428, "train_balanced_accuracy": 0.7044, "balanced_accuracy": 0.715, "F1": 0.7202, "precision": 0.7072, "recall": 0.7336 }, "microwave": { "cat_id": 78, "pos_dim": 342, "neg_dim": 573, "n_balanced": 108, "train_balanced_accuracy": 0.8995, "balanced_accuracy": 0.8889, "F1": 0.8909, "precision": 0.875, "recall": 0.9074 }, "oven": { "cat_id": 79, "pos_dim": 483, "neg_dim": 573, "n_balanced": 230, "train_balanced_accuracy": 0.8959, "balanced_accuracy": 0.9, "F1": 0.8987, "precision": 0.9107, "recall": 0.887 }, "sink": { "cat_id": 81, "pos_dim": 432, "neg_dim": 573, "n_balanced": 374, "train_balanced_accuracy": 0.8824, "balanced_accuracy": 0.9064, "F1": 0.9096, "precision": 0.88, "recall": 0.9412 }, "refrigerator": { "cat_id": 82, "pos_dim": 46, "neg_dim": 240, "n_balanced": 202, "train_balanced_accuracy": 0.8248, "balanced_accuracy": 0.8614, "F1": 0.8739, "precision": 0.8017, "recall": 0.9604 }, "book": { "cat_id": 84, "pos_dim": 185, "neg_dim": 240, "n_balanced": 460, "train_balanced_accuracy": 0.7777, "balanced_accuracy": 0.75, "F1": 0.7013, "precision": 0.871, "recall": 0.587 }, "clock": { "cat_id": 85, "pos_dim": 106, "neg_dim": 734, "n_balanced": 408, "train_balanced_accuracy": 0.7609, "balanced_accuracy": 0.7696, "F1": 0.7552, "precision": 0.8056, "recall": 0.7108 }, "vase": { "cat_id": 86, "pos_dim": 642, "neg_dim": 294, "n_balanced": 274, "train_balanced_accuracy": 0.7897, "balanced_accuracy": 0.7737, "F1": 0.7919, "precision": 0.7329, "recall": 0.8613 }, "scissors": { "cat_id": 87, "pos_dim": 648, "neg_dim": 369, "n_balanced": 56, "train_balanced_accuracy": 0.7909, "balanced_accuracy": 0.8214, "F1": 0.8333, "precision": 0.7812, "recall": 0.8929 }, "teddy bear": { "cat_id": 88, "pos_dim": 137, "neg_dim": 548, "n_balanced": 188, "train_balanced_accuracy": 0.8294, "balanced_accuracy": 0.8564, "F1": 0.8508, "precision": 0.8851, "recall": 0.8191 }, "toothbrush": { "cat_id": 90, "pos_dim": 650, "neg_dim": 548, "n_balanced": 68, "train_balanced_accuracy": 0.8203, "balanced_accuracy": 0.8676, "F1": 0.8657, "precision": 0.8788, "recall": 0.8529 } }, "distribution": { "n_categories": 78, "median": 0.8265, "min": 0.6702, "max": 0.9653, "at_least": { "0.90": 14, "0.85": 31, "0.80": 51, "0.75": 69, "0.70": 75 } } }