benchmark: sam3_2026-02-04T14-06-01.082997Z

README.md

@@ -36,9 +36,9 @@ manako:
   - name: detections
     type: detections
     description: List of detections
-  evaluation_score: 0.7690604340214497
+  evaluation_score: 0.729435525370383
   last_benchmark:
-    type: detect-iou-v1
-    ran_at: '2026-02-03T14:00:26.700082Z'
-    result_path: benchmark/results/2026-02-03T14-00-26.700105Z.json
+    type: sam3dataset-v1
+    ran_at: '2026-02-04T14:06:39.406509Z'
+    result_path: benchmark/sam3dataset/results/2026-02-04T14-06-39.405656Z.json
 ---

benchmark/sam3dataset/predictions/2026-02-04T14-06-39.405656Z.jsonl

@@ -0,0 +1,10 @@
+{"image_id": 209, "image_key": "saco-gold/images/metaclip_2_100002_2e1d98c0cedb00b4571e2ad2_jpeg.rf.ac92ffce1fef75a365ce4b296f1db04e.jpg", "predictions": [{"frame_idx": 0, "class": "person", "bbox": [239.88714599609375, 20.20615577697754, 406.8285827636719, 576.3162231445312], "score": 0.8060612678527832, "track_id": "f0-d0"}, {"frame_idx": 0, "class": "person", "bbox": [47.32451629638672, 15.893669128417969, 265.52581787109375, 574.5195922851562], "score": 0.7588138580322266, "track_id": "f0-d1"}, {"frame_idx": 0, "class": "person", "bbox": [0.3007650375366211, 20.398326873779297, 112.6234130859375, 481.52215576171875], "score": 0.6549990177154541, "track_id": "f0-d2"}, {"frame_idx": 0, "class": "person", "bbox": [244.78988647460938, 13.58382797241211, 279.3552551269531, 86.87496185302734], "score": 0.373250275850296, "track_id": "f0-d3"}], "expected_counts": {"barcode": 0, "car": 0, "cardboard box": 0, "fire": 0, "forklift": 0, "freight container": 0, "Gloves": 0, "helmet": 0, "ladder": 0, "license plate": 0, "person": 17, "qr code": 0, "road sign": 0, "safety vest": 0, "smoke": 0, "traffic cone": 0, "traffic light": 0, "truck": 0, "van": 0, "wood pallet": 0}, "iou": 0.8403196197196299, "per_class_iou": {"barcode": 0.0, "car": 0.0, "cardboard box": 0.0, "fire": 0.0, "forklift": 0.0, "freight container": 0.0, "Gloves": 0.0, "helmet": 0.0, "ladder": 0.0, "license plate": 0.0, "person": 0.8403196197196299, "qr code": 0.0, "road sign": 0.0, "safety vest": 0.0, "smoke": 0.0, "traffic cone": 0.0, "traffic light": 0.0, "truck": 0.0, "van": 0.0, "wood pallet": 0.0}}
+{"image_id": 477, "image_key": "saco-gold/images/metaclip_2_100002_51fb0d13147469b77518aaf4_jpeg.rf.5a647d09205446e113faad139c277d5e.jpg", "predictions": [{"frame_idx": 0, "class": "car", "bbox": [143.52386474609375, 78.30491638183594, 698.805908203125, 542.325439453125], "score": 0.9147508144378662, "track_id": "f0-d0"}, {"frame_idx": 0, "class": "car", "bbox": [0.22705078125, 144.31716918945312, 102.73274993896484, 286.47491455078125], "score": 0.2910258173942566, "track_id": "f0-d1"}], "expected_counts": {"barcode": 0, "car": 6, "cardboard box": 0, "fire": 0, "forklift": 0, "freight container": 0, "Gloves": 0, "helmet": 0, "ladder": 0, "license plate": 0, "person": 0, "qr code": 0, "road sign": 0, "safety vest": 0, "smoke": 0, "traffic cone": 0, "traffic light": 0, "truck": 0, "van": 0, "wood pallet": 0}, "iou": 0.9295584145522278, "per_class_iou": {"barcode": 0.0, "car": 0.9295584145522278, "cardboard box": 0.0, "fire": 0.0, "forklift": 0.0, "freight container": 0.0, "Gloves": 0.0, "helmet": 0.0, "ladder": 0.0, "license plate": 0.0, "person": 0.0, "qr code": 0.0, "road sign": 0.0, "safety vest": 0.0, "smoke": 0.0, "traffic cone": 0.0, "traffic light": 0.0, "truck": 0.0, "van": 0.0, "wood pallet": 0.0}}
+{"image_id": 132, "image_key": "saco-gold/images/metaclip_2_100002_55255beda3e5b364187ee921_jpeg.rf.b433bf24de9d435702f90c7f3e3687f9.jpg", "predictions": [{"frame_idx": 0, "class": "person", "bbox": [623.9235229492188, 169.8321533203125, 795.4994506835938, 613.0396728515625], "score": 0.8498489856719971, "track_id": "f0-d0"}, {"frame_idx": 0, "class": "person", "bbox": [57.45185852050781, 195.08575439453125, 490.4998779296875, 683.7188110351562], "score": 0.6958703994750977, "track_id": "f0-d1"}, {"frame_idx": 0, "class": "person", "bbox": [414.01617431640625, 118.6598129272461, 617.0319213867188, 667.8418579101562], "score": 0.6591689586639404, "track_id": "f0-d2"}, {"frame_idx": 0, "class": "helmet", "bbox": [428.04656982421875, 28.2147216796875, 574.3399658203125, 238.00035095214844], "score": 0.2943231463432312, "track_id": "f0-d3"}, {"frame_idx": 0, "class": "person", "bbox": [315.19696044921875, 173.63583374023438, 486.8261413574219, 588.0691528320312], "score": 0.2934528589248657, "track_id": "f0-d4"}, {"frame_idx": 0, "class": "person", "bbox": [58.95979690551758, 174.7000732421875, 357.30718994140625, 680.24267578125], "score": 0.25036031007766724, "track_id": "f0-d5"}], "expected_counts": {"barcode": 0, "car": 0, "cardboard box": 0, "fire": 0, "forklift": 0, "freight container": 0, "Gloves": 0, "helmet": 0, "ladder": 0, "license plate": 0, "person": 5, "qr code": 0, "road sign": 0, "safety vest": 0, "smoke": 0, "traffic cone": 0, "traffic light": 0, "truck": 0, "van": 0, "wood pallet": 0}, "iou": 0.7592161948787054, "per_class_iou": {"barcode": 0.0, "car": 0.0, "cardboard box": 0.0, "fire": 0.0, "forklift": 0.0, "freight container": 0.0, "Gloves": 0.0, "helmet": 0.7229831972149103, "ladder": 0.0, "license plate": 0.0, "person": 0.7664627944114644, "qr code": 0.0, "road sign": 0.0, "safety vest": 0.0, "smoke": 0.0, "traffic cone": 0.0, "traffic light": 0.0, "truck": 0.0, "van": 0.0, "wood pallet": 0.0}}
+{"image_id": 199, "image_key": "saco-gold/images/metaclip_2_100002_778fb5dfec8b564280555a51_jpeg.rf.e49c6a37959285bda44fff2751e1d9bd.jpg", "predictions": [{"frame_idx": 0, "class": "helmet", "bbox": [174.44329833984375, 286.32977294921875, 192.7139892578125, 300.57879638671875], "score": 0.2775278687477112, "track_id": "f0-d0"}], "expected_counts": {"barcode": 0, "car": 0, "cardboard box": 0, "fire": 0, "forklift": 0, "freight container": 0, "Gloves": 0, "helmet": 0, "ladder": 0, "license plate": 0, "person": 8, "qr code": 0, "road sign": 0, "safety vest": 0, "smoke": 0, "traffic cone": 0, "traffic light": 0, "truck": 0, "van": 0, "wood pallet": 0}, "iou": 0.306196070651317, "per_class_iou": {"barcode": 0.0, "car": 0.0, "cardboard box": 0.0, "fire": 0.0, "forklift": 0.0, "freight container": 0.0, "Gloves": 0.0, "helmet": 0.306196070651317, "ladder": 0.0, "license plate": 0.0, "person": 0.0, "qr code": 0.0, "road sign": 0.0, "safety vest": 0.0, "smoke": 0.0, "traffic cone": 0.0, "traffic light": 0.0, "truck": 0.0, "van": 0.0, "wood pallet": 0.0}}
+{"image_id": 44, "image_key": "saco-gold/images/metaclip_2_100002_7deed25ee9f0b5352df379e3_jpeg.rf.6c81db8863c048b0c7e2db843bdb87bd.jpg", "predictions": [{"frame_idx": 0, "class": "person", "bbox": [515.35302734375, 1331.4837646484375, 704.2612915039062, 1640.4334716796875], "score": 0.8610184192657471, "track_id": "f0-d0"}, {"frame_idx": 0, "class": "person", "bbox": [129.59828186035156, 1022.8570556640625, 346.669921875, 1292.41162109375], "score": 0.7971668243408203, "track_id": "f0-d1"}, {"frame_idx": 0, "class": "person", "bbox": [1.937678575515747, 1295.996337890625, 244.28030395507812, 1641.393310546875], "score": 0.7758616209030151, "track_id": "f0-d2"}, {"frame_idx": 0, "class": "person", "bbox": [508.9010009765625, 1018.080078125, 751.166015625, 1293.662353515625], "score": 0.7724421620368958, "track_id": "f0-d3"}, {"frame_idx": 0, "class": "person", "bbox": [351.89031982421875, 1317.2005615234375, 507.6310729980469, 1642.6451416015625], "score": 0.7445335388183594, "track_id": "f0-d4"}, {"frame_idx": 0, "class": "person", "bbox": [194.85914611816406, 731.8135986328125, 418.7989807128906, 1007.1458129882812], "score": 0.6990341544151306, "track_id": "f0-d5"}, {"frame_idx": 0, "class": "safety vest", "bbox": [68.28157806396484, 119.31956481933594, 306.0667724609375, 351.77935791015625], "score": 0.4842202663421631, "track_id": "f0-d6"}, {"frame_idx": 0, "class": "person", "bbox": [247.10459899902344, 1293.289794921875, 370.9768371582031, 1590.9404296875], "score": 0.4773564338684082, "track_id": "f0-d7"}, {"frame_idx": 0, "class": "person", "bbox": [64.17445373535156, 23.742328643798828, 427.0275573730469, 350.7831726074219], "score": 0.44771164655685425, "track_id": "f0-d8"}, {"frame_idx": 0, "class": "helmet", "bbox": [181.4456024169922, 1023.6815795898438, 260.3591003417969, 1102.284912109375], "score": 0.3565187454223633, "track_id": "f0-d9"}, {"frame_idx": 0, "class": "person", "bbox": [68.48941040039062, 25.98058319091797, 309.9913330078125, 352.15167236328125], "score": 0.33321458101272583, "track_id": "f0-d10"}], "expected_counts": {"barcode": 0, "car": 0, "cardboard box": 0, "fire": 0, "forklift": 0, "freight container": 0, "Gloves": 0, "helmet": 0, "ladder": 0, "license plate": 0, "person": 14, "qr code": 0, "road sign": 0, "safety vest": 0, "smoke": 0, "traffic cone": 0, "traffic light": 0, "truck": 0, "van": 0, "wood pallet": 0}, "iou": 0.8228505590624692, "per_class_iou": {"barcode": 0.0, "car": 0.0, "cardboard box": 0.0, "fire": 0.0, "forklift": 0.0, "freight container": 0.0, "Gloves": 0.0, "helmet": 0.7787724735014123, "ladder": 0.0, "license plate": 0.0, "person": 0.8197508171545036, "qr code": 0.0, "road sign": 0.0, "safety vest": 0.8948263217952157, "smoke": 0.0, "traffic cone": 0.0, "traffic light": 0.0, "truck": 0.0, "van": 0.0, "wood pallet": 0.0}}
+{"image_id": 533, "image_key": "saco-gold/images/metaclip_2_10002_532b5e81dea0eef2e0371605_jpeg.rf.8139de622393c6cdcfc24e1fac90c224.jpg", "predictions": [{"frame_idx": 0, "class": "person", "bbox": [133.38241577148438, 5.317783355712891, 513.8139038085938, 628.0], "score": 0.5105494260787964, "track_id": "f0-d0"}], "expected_counts": {"barcode": 0, "car": 0, "cardboard box": 0, "fire": 0, "forklift": 0, "freight container": 0, "Gloves": 0, "helmet": 0, "ladder": 0, "license plate": 0, "person": 6, "qr code": 0, "road sign": 0, "safety vest": 0, "smoke": 0, "traffic cone": 0, "traffic light": 0, "truck": 0, "van": 0, "wood pallet": 0}, "iou": 0.8975890285094377, "per_class_iou": {"barcode": 0.0, "car": 0.0, "cardboard box": 0.0, "fire": 0.0, "forklift": 0.0, "freight container": 0.0, "Gloves": 0.0, "helmet": 0.0, "ladder": 0.0, "license plate": 0.0, "person": 0.8975890285094377, "qr code": 0.0, "road sign": 0.0, "safety vest": 0.0, "smoke": 0.0, "traffic cone": 0.0, "traffic light": 0.0, "truck": 0.0, "van": 0.0, "wood pallet": 0.0}}
+{"image_id": 310, "image_key": "saco-gold/images/metaclip_2_10002_820eae69e727e6e53041f38a_jpeg.rf.a7a08ae2406ca257fa9f619763462eb0.jpg", "predictions": [{"frame_idx": 0, "class": "person", "bbox": [1103.853271484375, 426.12640380859375, 1130.521240234375, 496.11907958984375], "score": 0.27819013595581055, "track_id": "f0-d0"}], "expected_counts": {"barcode": 0, "car": 8, "cardboard box": 0, "fire": 0, "forklift": 0, "freight container": 0, "Gloves": 0, "helmet": 0, "ladder": 0, "license plate": 0, "person": 0, "qr code": 0, "road sign": 0, "safety vest": 0, "smoke": 0, "traffic cone": 0, "traffic light": 0, "truck": 0, "van": 0, "wood pallet": 0}, "iou": 0.206549711660426, "per_class_iou": {"barcode": 0.0, "car": 0.0, "cardboard box": 0.0, "fire": 0.0, "forklift": 0.0, "freight container": 0.0, "Gloves": 0.0, "helmet": 0.0, "ladder": 0.0, "license plate": 0.0, "person": 0.206549711660426, "qr code": 0.0, "road sign": 0.0, "safety vest": 0.0, "smoke": 0.0, "traffic cone": 0.0, "traffic light": 0.0, "truck": 0.0, "van": 0.0, "wood pallet": 0.0}}
+{"image_id": 28, "image_key": "saco-gold/images/metaclip_2_100002_2af4b52eb11d09506ac14dff_jpeg.rf.f36be955497c5574d68d6f462b6111dc.jpg", "predictions": [{"frame_idx": 0, "class": "car", "bbox": [177.9336395263672, 272.34259033203125, 1086.307861328125, 655.3825073242188], "score": 0.9260011911392212, "track_id": "f0-d0"}], "expected_counts": {"barcode": 0, "car": 0, "cardboard box": 0, "fire": 0, "forklift": 0, "freight container": 0, "Gloves": 0, "helmet": 0, "ladder": 0, "license plate": 0, "person": 0, "qr code": 0, "road sign": 5, "safety vest": 0, "smoke": 0, "traffic cone": 0, "traffic light": 0, "truck": 0, "van": 0, "wood pallet": 0}, "iou": 0.9719075537082313, "per_class_iou": {"barcode": 0.0, "car": 0.9719075537082313, "cardboard box": 0.0, "fire": 0.0, "forklift": 0.0, "freight container": 0.0, "Gloves": 0.0, "helmet": 0.0, "ladder": 0.0, "license plate": 0.0, "person": 0.0, "qr code": 0.0, "road sign": 0.0, "safety vest": 0.0, "smoke": 0.0, "traffic cone": 0.0, "traffic light": 0.0, "truck": 0.0, "van": 0.0, "wood pallet": 0.0}}
+{"image_id": 443, "image_key": "saco-gold/images/metaclip_2_100002_7884da9db2c330c87d224f66_jpeg.rf.94b9aa2e1991c430eed4811e6179c193.jpg", "predictions": [{"frame_idx": 0, "class": "car", "bbox": [147.56568908691406, 60.360107421875, 743.2879638671875, 535.3960571289062], "score": 0.8668753504753113, "track_id": "f0-d0"}], "expected_counts": {"barcode": 0, "car": 5, "cardboard box": 0, "fire": 0, "forklift": 0, "freight container": 0, "Gloves": 0, "helmet": 0, "ladder": 0, "license plate": 0, "person": 0, "qr code": 0, "road sign": 0, "safety vest": 0, "smoke": 0, "traffic cone": 0, "traffic light": 0, "truck": 0, "van": 0, "wood pallet": 0}, "iou": 0.9405826125486482, "per_class_iou": {"barcode": 0.0, "car": 0.9405826125486482, "cardboard box": 0.0, "fire": 0.0, "forklift": 0.0, "freight container": 0.0, "Gloves": 0.0, "helmet": 0.0, "ladder": 0.0, "license plate": 0.0, "person": 0.0, "qr code": 0.0, "road sign": 0.0, "safety vest": 0.0, "smoke": 0.0, "traffic cone": 0.0, "traffic light": 0.0, "truck": 0.0, "van": 0.0, "wood pallet": 0.0}}
+{"image_id": 457, "image_key": "saco-gold/images/metaclip_2_100002_c98ec610150e52b1c2a97c41_jpeg.rf.312de5fcabc56453af07a245c7cec170.jpg", "predictions": [{"frame_idx": 0, "class": "person", "bbox": [71.69149017333984, 197.6784210205078, 398.08367919921875, 582.3663330078125], "score": 0.8216882348060608, "track_id": "f0-d0"}, {"frame_idx": 0, "class": "person", "bbox": [745.1033935546875, 182.343994140625, 941.1646728515625, 566.1421508789062], "score": 0.7821482419967651, "track_id": "f0-d1"}, {"frame_idx": 0, "class": "person", "bbox": [473.697265625, 203.32870483398438, 658.7376708984375, 581.9708862304688], "score": 0.7167372703552246, "track_id": "f0-d2"}, {"frame_idx": 0, "class": "person", "bbox": [544.609130859375, 190.61044311523438, 605.6011962890625, 252.86026000976562], "score": 0.25140002369880676, "track_id": "f0-d3"}], "expected_counts": {"barcode": 0, "car": 0, "cardboard box": 0, "fire": 0, "forklift": 0, "freight container": 0, "Gloves": 0, "helmet": 0, "ladder": 0, "license plate": 0, "person": 5, "qr code": 0, "road sign": 0, "safety vest": 0, "smoke": 0, "traffic cone": 0, "traffic light": 0, "truck": 0, "van": 0, "wood pallet": 0}, "iou": 0.6195854884127385, "per_class_iou": {"barcode": 0.0, "car": 0.0, "cardboard box": 0.0, "fire": 0.0, "forklift": 0.0, "freight container": 0.0, "Gloves": 0.0, "helmet": 0.0, "ladder": 0.0, "license plate": 0.0, "person": 0.6195854884127385, "qr code": 0.0, "road sign": 0.0, "safety vest": 0.0, "smoke": 0.0, "traffic cone": 0.0, "traffic light": 0.0, "truck": 0.0, "van": 0.0, "wood pallet": 0.0}}

benchmark/sam3dataset/results/2026-02-04T14-06-39.405656Z.json

@@ -0,0 +1,91 @@
+{
+  "meta": {
+    "element_id": "MTerryJack/Element-Detect-WarehouseLogistics",
+    "classes": [
+      "barcode",
+      "car",
+      "cardboard box",
+      "fire",
+      "forklift",
+      "freight container",
+      "Gloves",
+      "helmet",
+      "ladder",
+      "license plate",
+      "person",
+      "qr code",
+      "road sign",
+      "safety vest",
+      "smoke",
+      "traffic cone",
+      "traffic light",
+      "truck",
+      "van",
+      "wood pallet"
+    ],
+    "max_images": 10,
+    "matched_images": 10
+  },
+  "results": {
+    "overall_iou": 0.729435525370383,
+    "per_class_iou": {
+      "barcode": 0.0,
+      "car": 0.2842048580809107,
+      "cardboard box": 0.0,
+      "fire": 0.0,
+      "forklift": 0.0,
+      "freight container": 0.0,
+      "Gloves": 0.0,
+      "helmet": 0.18079517413676396,
+      "ladder": 0.0,
+      "license plate": 0.0,
+      "person": 0.41502574598682,
+      "qr code": 0.0,
+      "road sign": 0.0,
+      "safety vest": 0.08948263217952157,
+      "smoke": 0.0,
+      "traffic cone": 0.0,
+      "traffic light": 0.0,
+      "truck": 0.0,
+      "van": 0.0,
+      "wood pallet": 0.0
+    },
+    "matched_images": 10,
+    "gt_count": 720,
+    "pred_count": 32,
+    "classes": [
+      "barcode",
+      "car",
+      "cardboard box",
+      "fire",
+      "forklift",
+      "freight container",
+      "Gloves",
+      "helmet",
+      "ladder",
+      "license plate",
+      "person",
+      "qr code",
+      "road sign",
+      "safety vest",
+      "smoke",
+      "traffic cone",
+      "traffic light",
+      "truck",
+      "van",
+      "wood pallet"
+    ]
+  },
+  "selected": [
+    "saco-gold/images/metaclip_2_100002_2e1d98c0cedb00b4571e2ad2_jpeg.rf.ac92ffce1fef75a365ce4b296f1db04e.jpg",
+    "saco-gold/images/metaclip_2_100002_7deed25ee9f0b5352df379e3_jpeg.rf.6c81db8863c048b0c7e2db843bdb87bd.jpg",
+    "saco-gold/images/metaclip_2_100002_778fb5dfec8b564280555a51_jpeg.rf.e49c6a37959285bda44fff2751e1d9bd.jpg",
+    "saco-gold/images/metaclip_2_10002_820eae69e727e6e53041f38a_jpeg.rf.a7a08ae2406ca257fa9f619763462eb0.jpg",
+    "saco-gold/images/metaclip_2_100002_51fb0d13147469b77518aaf4_jpeg.rf.5a647d09205446e113faad139c277d5e.jpg",
+    "saco-gold/images/metaclip_2_10002_532b5e81dea0eef2e0371605_jpeg.rf.8139de622393c6cdcfc24e1fac90c224.jpg",
+    "saco-gold/images/metaclip_2_100002_2af4b52eb11d09506ac14dff_jpeg.rf.f36be955497c5574d68d6f462b6111dc.jpg",
+    "saco-gold/images/metaclip_2_100002_55255beda3e5b364187ee921_jpeg.rf.b433bf24de9d435702f90c7f3e3687f9.jpg",
+    "saco-gold/images/metaclip_2_100002_7884da9db2c330c87d224f66_jpeg.rf.94b9aa2e1991c430eed4811e6179c193.jpg",
+    "saco-gold/images/metaclip_2_100002_c98ec610150e52b1c2a97c41_jpeg.rf.312de5fcabc56453af07a245c7cec170.jpg"
+  ]
+}