benchmark: sam3_2026-02-06T16-53-16.986450Z

README.md

@@ -40,9 +40,9 @@ manako:
   - name: detections
     type: detections
     description: List of detections
-  evaluation_score: 0.7449190139850637
+  evaluation_score: 0.40845817464489526
   last_benchmark:
-    type: detect-iou-v1
-    ran_at: '2026-02-06T16:16:13.772410Z'
-    result_path: benchmark/results/2026-02-06T16-16-13.772431Z.json
+    type: sam3dataset-v1
+    ran_at: '2026-02-06T16:53:45.586835Z'
+    result_path: benchmark/sam3dataset/results/2026-02-06T16-53-45.583070Z.json
 ---

benchmark/sam3dataset/predictions/2026-02-06T16-53-45.583070Z.jsonl

@@ -0,0 +1,5 @@
+{"image_id": 318, "image_key": "saco-gold/images/metaclip_2_10002_254798ef531a2c94dc8e4fd7_jpeg.rf.35bebfe7ce93d0806b23cca8286c96b2.jpg", "predictions": [{"frame_idx": 0, "class": "person", "bbox": [400.1846923828125, 306.9697570800781, 418.05029296875, 364.8869323730469], "score": 0.5832783579826355, "track_id": "f0-d0"}, {"frame_idx": 0, "class": "person", "bbox": [332.5762939453125, 321.1940002441406, 352.69500732421875, 399.8028564453125], "score": 0.4651326537132263, "track_id": "f0-d1"}, {"frame_idx": 0, "class": "person", "bbox": [333.3778381347656, 322.5999755859375, 352.21160888671875, 370.23663330078125], "score": 0.42194825410842896, "track_id": "f0-d2"}, {"frame_idx": 0, "class": "person", "bbox": [366.98712158203125, 318.8979187011719, 382.1539611816406, 361.2187194824219], "score": 0.3981613218784332, "track_id": "f0-d3"}, {"frame_idx": 0, "class": "person", "bbox": [350.8757019042969, 320.6013488769531, 365.02203369140625, 361.67236328125], "score": 0.35006654262542725, "track_id": "f0-d4"}, {"frame_idx": 0, "class": "person", "bbox": [357.2750549316406, 316.8930358886719, 368.5097961425781, 362.34332275390625], "score": 0.3126699924468994, "track_id": "f0-d5"}, {"frame_idx": 0, "class": "person", "bbox": [387.3915100097656, 319.49176025390625, 400.2696838378906, 360.4873352050781], "score": 0.3040831685066223, "track_id": "f0-d6"}, {"frame_idx": 0, "class": "person", "bbox": [282.1150817871094, 322.6817932128906, 303.5845947265625, 380.8721923828125], "score": 0.2825131416320801, "track_id": "f0-d7"}, {"frame_idx": 0, "class": "person", "bbox": [364.3729248046875, 318.53704833984375, 379.0445556640625, 362.1617736816406], "score": 0.26479005813598633, "track_id": "f0-d8"}], "expected_counts": {"a streetlight": 11}, "iou": 0.0, "per_class_iou": {"a 2D barcode": 0.0, "car": 0.0, "cardboard box": 0.0, "fire pot": 0.0, "the fork handle": 0.0, "a plastic container": 0.0, "Gloves": 0.0, "the silver helmet": 0.0, "a ladder": 0.0, "yellow license plate": 0.0, "person": 0.0, "the QR code": 0.0, "road sign": 0.0, "vest": 0.0, "the road surface": 0.0, "a streetlight": 0.0, "a truck": 0.0, "a white van": 0.0, "wooden pallet": 0.0}}
+{"image_id": 210, "image_key": "saco-gold/images/metaclip_2_100002_2e1d98c0cedb00b4571e2ad2_jpeg.rf.ac92ffce1fef75a365ce4b296f1db04e.jpg", "predictions": [{"frame_idx": 0, "class": "person", "bbox": [239.88714599609375, 20.20615577697754, 406.8285827636719, 576.3162231445312], "score": 0.8060612678527832, "track_id": "f0-d0"}, {"frame_idx": 0, "class": "person", "bbox": [47.32451629638672, 15.893669128417969, 265.52581787109375, 574.5195922851562], "score": 0.7588138580322266, "track_id": "f0-d1"}, {"frame_idx": 0, "class": "person", "bbox": [0.3007650375366211, 20.398326873779297, 112.6234130859375, 481.52215576171875], "score": 0.6549990177154541, "track_id": "f0-d2"}, {"frame_idx": 0, "class": "person", "bbox": [244.78988647460938, 13.58382797241211, 279.3552551269531, 86.87496185302734], "score": 0.373250275850296, "track_id": "f0-d3"}], "expected_counts": {"person": 17}, "iou": 0.8841604966663094, "per_class_iou": {"a 2D barcode": 0.0, "car": 0.0, "cardboard box": 0.0, "fire pot": 0.0, "the fork handle": 0.0, "a plastic container": 0.0, "Gloves": 0.0, "the silver helmet": 0.0, "a ladder": 0.0, "yellow license plate": 0.0, "person": 0.8841604966663094, "the QR code": 0.0, "road sign": 0.0, "vest": 0.0, "the road surface": 0.0, "a streetlight": 0.0, "a truck": 0.0, "a white van": 0.0, "wooden pallet": 0.0}}
+{"image_id": 200, "image_key": "saco-gold/images/metaclip_2_100002_778fb5dfec8b564280555a51_jpeg.rf.e49c6a37959285bda44fff2751e1d9bd.jpg", "predictions": [{"frame_idx": 0, "class": "helmet", "bbox": [174.44329833984375, 286.32977294921875, 192.7139892578125, 300.57879638671875], "score": 0.2775278687477112, "track_id": "f0-d0"}], "expected_counts": {"person": 8}, "iou": 0.22755567594188467, "per_class_iou": {"a 2D barcode": 0.0, "car": 0.0, "cardboard box": 0.0, "fire pot": 0.0, "the fork handle": 0.0, "a plastic container": 0.0, "Gloves": 0.0, "the silver helmet": 0.0, "a ladder": 0.0, "yellow license plate": 0.0, "person": 0.0, "the QR code": 0.0, "road sign": 0.0, "vest": 0.0, "the road surface": 0.0, "a streetlight": 0.0, "a truck": 0.0, "a white van": 0.0, "wooden pallet": 0.0}}
+{"image_id": 311, "image_key": "saco-gold/images/metaclip_2_10002_820eae69e727e6e53041f38a_jpeg.rf.a7a08ae2406ca257fa9f619763462eb0.jpg", "predictions": [{"frame_idx": 0, "class": "person", "bbox": [1103.853271484375, 426.12640380859375, 1130.521240234375, 496.11907958984375], "score": 0.27819013595581055, "track_id": "f0-d0"}], "expected_counts": {"car": 8}, "iou": 0.0, "per_class_iou": {"a 2D barcode": 0.0, "car": 0.0, "cardboard box": 0.0, "fire pot": 0.0, "the fork handle": 0.0, "a plastic container": 0.0, "Gloves": 0.0, "the silver helmet": 0.0, "a ladder": 0.0, "yellow license plate": 0.0, "person": 0.0, "the QR code": 0.0, "road sign": 0.0, "vest": 0.0, "the road surface": 0.0, "a streetlight": 0.0, "a truck": 0.0, "a white van": 0.0, "wooden pallet": 0.0}}
+{"image_id": 45, "image_key": "saco-gold/images/metaclip_2_100002_7deed25ee9f0b5352df379e3_jpeg.rf.6c81db8863c048b0c7e2db843bdb87bd.jpg", "predictions": [{"frame_idx": 0, "class": "person", "bbox": [515.35302734375, 1331.4837646484375, 704.2612915039062, 1640.4334716796875], "score": 0.8610184192657471, "track_id": "f0-d0"}, {"frame_idx": 0, "class": "person", "bbox": [129.59828186035156, 1022.8570556640625, 346.669921875, 1292.41162109375], "score": 0.7971668243408203, "track_id": "f0-d1"}, {"frame_idx": 0, "class": "person", "bbox": [1.937678575515747, 1295.996337890625, 244.28030395507812, 1641.393310546875], "score": 0.7758616209030151, "track_id": "f0-d2"}, {"frame_idx": 0, "class": "person", "bbox": [508.9010009765625, 1018.080078125, 751.166015625, 1293.662353515625], "score": 0.7724421620368958, "track_id": "f0-d3"}, {"frame_idx": 0, "class": "person", "bbox": [351.89031982421875, 1317.2005615234375, 507.6310729980469, 1642.6451416015625], "score": 0.7445335388183594, "track_id": "f0-d4"}, {"frame_idx": 0, "class": "person", "bbox": [194.85914611816406, 731.8135986328125, 418.7989807128906, 1007.1458129882812], "score": 0.6990341544151306, "track_id": "f0-d5"}, {"frame_idx": 0, "class": "safety vest", "bbox": [68.28157806396484, 119.31956481933594, 306.0667724609375, 351.77935791015625], "score": 0.4842202663421631, "track_id": "f0-d6"}, {"frame_idx": 0, "class": "person", "bbox": [247.10459899902344, 1293.289794921875, 370.9768371582031, 1590.9404296875], "score": 0.4773564338684082, "track_id": "f0-d7"}, {"frame_idx": 0, "class": "person", "bbox": [64.17445373535156, 23.742328643798828, 427.0275573730469, 350.7831726074219], "score": 0.44771164655685425, "track_id": "f0-d8"}, {"frame_idx": 0, "class": "helmet", "bbox": [181.4456024169922, 1023.6815795898438, 260.3591003417969, 1102.284912109375], "score": 0.3565187454223633, "track_id": "f0-d9"}, {"frame_idx": 0, "class": "person", "bbox": [68.48941040039062, 25.98058319091797, 309.9913330078125, 352.15167236328125], "score": 0.33321458101272583, "track_id": "f0-d10"}], "expected_counts": {"person": 14}, "iou": 0.9305747006162823, "per_class_iou": {"a 2D barcode": 0.0, "car": 0.0, "cardboard box": 0.0, "fire pot": 0.0, "the fork handle": 0.0, "a plastic container": 0.0, "Gloves": 0.0, "the silver helmet": 0.0, "a ladder": 0.0, "yellow license plate": 0.0, "person": 0.9305747006162823, "the QR code": 0.0, "road sign": 0.0, "vest": 0.0, "the road surface": 0.0, "a streetlight": 0.0, "a truck": 0.0, "a white van": 0.0, "wooden pallet": 0.0}}

benchmark/sam3dataset/results/2026-02-06T16-53-45.583070Z.json

@@ -0,0 +1,85 @@
+{
+  "meta": {
+    "element_id": "MTerryJack/Element-Detect-WarehouseLogistics",
+    "classes": [
+      "a 2D barcode",
+      "car",
+      "cardboard box",
+      "fire pot",
+      "the fork handle",
+      "a plastic container",
+      "Gloves",
+      "the silver helmet",
+      "a ladder",
+      "yellow license plate",
+      "person",
+      "the QR code",
+      "road sign",
+      "vest",
+      "fire pot",
+      "the road surface",
+      "a streetlight",
+      "a truck",
+      "a white van",
+      "wooden pallet"
+    ],
+    "max_images": 5,
+    "matched_images": 5
+  },
+  "results": {
+    "overall_iou": 0.40845817464489526,
+    "per_class_iou": {
+      "a 2D barcode": 0.0,
+      "car": 0.0,
+      "cardboard box": 0.0,
+      "fire pot": 0.0,
+      "the fork handle": 0.0,
+      "a plastic container": 0.0,
+      "Gloves": 0.0,
+      "the silver helmet": 0.0,
+      "a ladder": 0.0,
+      "yellow license plate": 0.0,
+      "person": 0.36294703945651835,
+      "the QR code": 0.0,
+      "road sign": 0.0,
+      "vest": 0.0,
+      "the road surface": 0.0,
+      "a streetlight": 0.0,
+      "a truck": 0.0,
+      "a white van": 0.0,
+      "wooden pallet": 0.0
+    },
+    "matched_images": 5,
+    "gt_count": 58,
+    "pred_count": 26,
+    "classes": [
+      "a 2D barcode",
+      "car",
+      "cardboard box",
+      "fire pot",
+      "the fork handle",
+      "a plastic container",
+      "Gloves",
+      "the silver helmet",
+      "a ladder",
+      "yellow license plate",
+      "person",
+      "the QR code",
+      "road sign",
+      "vest",
+      "fire pot",
+      "the road surface",
+      "a streetlight",
+      "a truck",
+      "a white van",
+      "wooden pallet"
+    ]
+  },
+  "selected": [
+    "saco-gold/images/metaclip_2_100002_2e1d98c0cedb00b4571e2ad2_jpeg.rf.ac92ffce1fef75a365ce4b296f1db04e.jpg",
+    "saco-gold/images/metaclip_2_100002_7deed25ee9f0b5352df379e3_jpeg.rf.6c81db8863c048b0c7e2db843bdb87bd.jpg",
+    "saco-gold/images/metaclip_2_10002_254798ef531a2c94dc8e4fd7_jpeg.rf.35bebfe7ce93d0806b23cca8286c96b2.jpg",
+    "saco-gold/images/metaclip_2_100002_778fb5dfec8b564280555a51_jpeg.rf.e49c6a37959285bda44fff2751e1d9bd.jpg",
+    "saco-gold/images/metaclip_2_10002_820eae69e727e6e53041f38a_jpeg.rf.a7a08ae2406ca257fa9f619763462eb0.jpg"
+  ]
+}