configs/evaluation/object_detection/OD06_OD07_coco2017_quality_eval.json

{
  "schema_version": "1.0",
  "evaluation_id": "OD06_OD07_COCO2017_VAL_FULL",
  "scope": "Exact acquired raw-head EfficientDet-Lite TFLite artifacts",
  "dataset": {
    "name": "COCO 2017",
    "split": "val2017",
    "expected_images": 5000,
    "expected_categories": 80,
    "image_archive": {
      "canonical_url": "http://images.cocodataset.org/zips/val2017.zip",
      "path": "research/downloads/coco2017/val2017.zip",
      "expected_bytes": 815585330,
      "expected_md5": "442b8da7639aecaf257c1dceb8ba8c80",
      "expected_sha256": "4f7e2ccb2866ec5041993c9cf2a952bbed69647b115d0f74da7ce8f4bef82f05"
    },
    "annotation_archive": {
      "canonical_url": "http://images.cocodataset.org/annotations/annotations_trainval2017.zip",
      "path": "research/downloads/coco2017/annotations_trainval2017.zip",
      "expected_bytes": 252907541,
      "expected_md5": "f4bbac642086de4f52a3fdda2de5fa2c",
      "expected_sha256": "113a836d90195ee1f884e704da6304dfaaecff1f023f49b6ca93c4aaae470268"
    },
    "image_directory": "research/downloads/coco2017/val2017",
    "annotation_path": "research/downloads/coco2017/annotations/instances_val2017.json",
    "annotation_sha256": "e8c7f7908f1d7278341fae127d0da654f102f11bd7b21d8aeefa635b8c810b6f",
    "license": {
      "annotations": "Creative Commons Attribution 4.0",
      "images": "Copyright remains with image owners; use must follow Flickr terms",
      "terms_path": "research/evidence/detection/sources/coco_terms_of_use.html",
      "terms_sha256": "bd019f88ee44c29b2f19c5b99888cf5bc2e7c16f57b6e52af8ed3a60462e8bdd",
      "redistribute_images": false
    }
  },
  "protocol": {
    "evaluator": "pycocotools.cocoeval.COCOeval",
    "iou_type": "bbox",
    "image_set": "all 5,000 public val2017 images",
    "max_detections": [1, 10, 100],
    "task_api_max_results": 100,
    "task_api_score_threshold": 0.0,
    "task_api_category_allowlist": "the 80 category names read from instances_val2017.json",
    "task_api_default_nms": {
      "overlap_type": "INTERSECTION_OVER_UNION",
      "min_suppression_threshold": 0.3,
      "algorithm": "DEFAULT",
      "multiclass_nms": false
    },
    "preprocessing": "MediaPipe ObjectDetector image preprocessing configured from each exact model's TFLite metadata",
    "postprocessing": "MediaPipe ObjectDetector graph consumes each exact model's DETECTOR_METADATA fixed anchors and tensor decoding options, applies its default out-of-model NMS, projects boxes to the original image, and maps embedded labels to COCO category names",
    "bbox_output": "[origin_x, origin_y, width, height] pixels in the original image coordinate system",
    "random_augmentation": false,
    "latency_measurement": false,
    "acceptance_threshold": null,
    "completion_status_meaning": "PASS means the exact artifact completed the full official metric calculation; it is not an accuracy-threshold claim"
  },
  "runtime": {
    "python": "environment/quality/od_coco/.venv/bin/python",
    "python_version": "3.12.13",
    "mediapipe_version": "0.10.31",
    "mediapipe_linux_x86_64_wheel_sha256": "360de896205822419b14d7574c2812e08946769228f7548c31dd871b150d9bcc",
    "mediapipe_source_revision": "cb0902f2dd9f773a30ce92c9331fcaf090f96d89",
    "pycocotools_version": "2.0 built from the official COCO API revision",
    "cocoapi_reference_revision": "8c9bcc3cf640524c4c20a9c40e89cb6a2f2fa0e9",
    "workers": 4,
    "worker_policy": "one independent MediaPipe ObjectDetector session per exact model variant"
  },
  "models": {
    "OD06": {
      "name": "EfficientDet-Lite0",
      "input_size": [320, 320],
      "expected_anchor_count": 19206,
      "expected_detector_metadata_sha256": "6168e1c05517ea700c206bfe6ac6bfe34e32644b715fcf9d6be64af63c3251a4",
      "variants": {
        "fp32": {
          "path": "research/downloads/detection/efficientdet_lite0_float32_v1.tflite",
          "source_url": "https://storage.googleapis.com/mediapipe-models/object_detector/efficientdet_lite0/float32/1/efficientdet_lite0.tflite",
          "expected_bytes": 13836895,
          "expected_sha256": "40338edf5ec70d43e318b0a716a84d4564cd1802759a7a07170c7e43796dbf58",
          "runtime_input_dtype": "FLOAT32"
        },
        "public_int8": {
          "path": "research/downloads/detection/efficientdet_lite0_int8_v1.tflite",
          "source_url": "https://storage.googleapis.com/mediapipe-models/object_detector/efficientdet_lite0/int8/1/efficientdet_lite0.tflite",
          "expected_bytes": 4602795,
          "expected_sha256": "0720bf247bd76e6594ea28fa9c6f7c5242be774818997dbbeffc4da460c723bb",
          "runtime_input_dtype": "UINT8"
        }
      },
      "published_context": {
        "metric": "COCO 2017 validation bbox mAP percent",
        "value": 25.69,
        "source_url": "https://www.kaggle.com/models/tensorflow/efficientdet/TfLite/lite0-detection-metadata",
        "source_path": "research/evidence/detection/sources/kaggle_efficientdet_model_api.json",
        "source_sha256": "b48d12c46ee2fdad0daeeaa58dcc4bf91cfc1b7dccb21ab32c73a0e71fbf05bd",
        "relationship": "Related postprocessed EfficientDet-Lite0 TFLite model variant, not byte-identical to either evaluated raw-head artifact",
        "exact_artifact_threshold": false,
        "use": "CONTEXT_ONLY"
      }
    },
    "OD07": {
      "name": "EfficientDet-Lite2",
      "input_size": [448, 448],
      "expected_anchor_count": 37629,
      "expected_detector_metadata_sha256": "a98460be5879f3c3f08415aab3c27f9c3272a3b9dbc4514239d8f712dece32ef",
      "variants": {
        "fp32": {
          "path": "research/downloads/detection/efficientdet_lite2_float32_v1.tflite",
          "source_url": "https://storage.googleapis.com/mediapipe-models/object_detector/efficientdet_lite2/float32/1/efficientdet_lite2.tflite",
          "expected_bytes": 23096891,
          "expected_sha256": "ad2abbf2b4e10585e15176fd7b5ef03c28dda959ae26fc142549fdd1814db91d",
          "runtime_input_dtype": "FLOAT32"
        },
        "public_int8": {
          "path": "research/downloads/detection/efficientdet_lite2_int8_v1.tflite",
          "source_url": "https://storage.googleapis.com/mediapipe-models/object_detector/efficientdet_lite2/int8/1/efficientdet_lite2.tflite",
          "expected_bytes": 7515971,
          "expected_sha256": "b3f50554cb0ea559e90328845f7d9ba4d13c8bff372914d24e06bc8bb72fa896",
          "runtime_input_dtype": "UINT8"
        }
      },
      "published_context": {
        "metric": "COCO 2017 validation bbox mAP percent",
        "value": 33.97,
        "source_url": "https://www.kaggle.com/models/tensorflow/efficientdet/TfLite/lite2-detection-metadata",
        "source_path": "research/evidence/detection/sources/kaggle_efficientdet_model_api.json",
        "source_sha256": "b48d12c46ee2fdad0daeeaa58dcc4bf91cfc1b7dccb21ab32c73a0e71fbf05bd",
        "relationship": "Related postprocessed EfficientDet-Lite2 TFLite model variant, not byte-identical to either evaluated raw-head artifact",
        "exact_artifact_threshold": false,
        "use": "CONTEXT_ONLY"
      }
    }
  },
  "authoritative_sources": [
    {
      "name": "mediapipe_object_detector_python",
      "url": "https://raw.githubusercontent.com/google-ai-edge/mediapipe/cb0902f2dd9f773a30ce92c9331fcaf090f96d89/mediapipe/tasks/python/vision/object_detector.py",
      "path": "research/evidence/detection/od06_od07_coco2017/sources/mediapipe_object_detector_cb0902f.py",
      "sha256": "7221da145cdb0ffb2d756835c259a4424f632f557d1ef0b84a621dd8cdbffaba"
    },
    {
      "name": "mediapipe_object_detector_cpp_options",
      "url": "https://raw.githubusercontent.com/google-ai-edge/mediapipe/cb0902f2dd9f773a30ce92c9331fcaf090f96d89/mediapipe/tasks/cc/vision/object_detector/object_detector.h",
      "path": "research/evidence/detection/od06_od07_coco2017/sources/mediapipe_object_detector_cb0902f.h",
      "sha256": "f9c2648d7e5820907533c84741b00cc00b857ed593c9ed8992708bbc6b88a5a8"
    },
    {
      "name": "mediapipe_object_detector_graph",
      "url": "https://raw.githubusercontent.com/google-ai-edge/mediapipe/cb0902f2dd9f773a30ce92c9331fcaf090f96d89/mediapipe/tasks/cc/vision/object_detector/object_detector_graph.cc",
      "path": "research/evidence/detection/od06_od07_coco2017/sources/mediapipe_object_detector_graph_cb0902f.cc",
      "sha256": "7ab98ba442f4ef5a9072171e86a0bf9383650d41234582c1e5d0e4b77539e809"
    },
    {
      "name": "mediapipe_detection_postprocessing_graph",
      "url": "https://raw.githubusercontent.com/google-ai-edge/mediapipe/cb0902f2dd9f773a30ce92c9331fcaf090f96d89/mediapipe/tasks/cc/components/processors/detection_postprocessing_graph.cc",
      "path": "research/evidence/detection/od06_od07_coco2017/sources/mediapipe_detection_postprocessing_graph_cb0902f.cc",
      "sha256": "538ed037b4ac2fbcfd6b54f26dbc8f8516dc22184ae0f1c5525788f6a4eae446"
    },
    {
      "name": "mediapipe_detector_options",
      "url": "https://raw.githubusercontent.com/google-ai-edge/mediapipe/cb0902f2dd9f773a30ce92c9331fcaf090f96d89/mediapipe/tasks/cc/components/processors/proto/detector_options.proto",
      "path": "research/evidence/detection/od06_od07_coco2017/sources/mediapipe_detector_options_cb0902f.proto",
      "sha256": "558423123299b07e0f0730b45c0d3059c50b320d71987bb5a72b5baebc995fef"
    },
    {
      "name": "mediapipe_object_detector_metadata_schema",
      "url": "https://raw.githubusercontent.com/google-ai-edge/mediapipe/cb0902f2dd9f773a30ce92c9331fcaf090f96d89/mediapipe/tasks/metadata/object_detector_metadata_schema.fbs",
      "path": "research/evidence/detection/od06_od07_coco2017/sources/mediapipe_object_detector_metadata_schema_cb0902f.fbs",
      "sha256": "2cc8719826de6212a31d367c6d2704372e2ea0a2bfa6cc31727a7663f6c84e22"
    },
    {
      "name": "cocoapi_cocoeval",
      "url": "https://raw.githubusercontent.com/cocodataset/cocoapi/8c9bcc3cf640524c4c20a9c40e89cb6a2f2fa0e9/PythonAPI/pycocotools/cocoeval.py",
      "path": "research/evidence/detection/od06_od07_coco2017/sources/cocoapi_cocoeval_8c9bcc3.py",
      "sha256": "e514af401848d5a4cc5d3512bd93d2c317f92272c6d30f218e0f00b7b4110534"
    },
    {
      "name": "cocoapi_coco",
      "url": "https://raw.githubusercontent.com/cocodataset/cocoapi/8c9bcc3cf640524c4c20a9c40e89cb6a2f2fa0e9/PythonAPI/pycocotools/coco.py",
      "path": "research/evidence/detection/od06_od07_coco2017/sources/cocoapi_coco_8c9bcc3.py",
      "sha256": "61b97f193862b48f8e4c61430dd15704794b44d68ae8284f12f8bfbcb420ebbe"
    }
  ],
  "prohibited_operations": {
    "training": false,
    "calibration": false,
    "ptq_or_qat": false,
    "model_conversion": false,
    "model_or_weight_modification": false,
    "latency_benchmark": false,
    "mlir_or_allocator_work": false
  }
}