| { |
| "schema_version": "1.0", |
| "model": { |
| "model_id": "AD01", |
| "model_name": "MLPerf Tiny Deep AutoEncoder (ToyCar)", |
| "task": "anomaly_detection", |
| "architecture_family": "Dense AutoEncoder", |
| "source_framework": "TensorFlow/Keras and TensorFlow Lite", |
| "source_repository": "https://github.com/mlcommons/tiny/tree/4addd0fa08d216e20637637874e084895f289da4/benchmark/training/anomaly_detection", |
| "license": "MIT", |
| "public_quantized_available": true, |
| "paired_fp32_available": true, |
| "pair_compatibility": "VERIFIED", |
| "dataset": "DCASE 2020 Task 2 ToyCar", |
| "metric": "AUC and partial AUC from reconstruction error", |
| "input_shape": [ |
| 1, |
| 640 |
| ], |
| "output_shape": [ |
| 1, |
| 640 |
| ], |
| "preprocessing": "Upstream MLPerf Tiny DCASE pipeline: 640-element normalized log-mel feature vector", |
| "label_space": "640-element reconstructed feature vector; anomaly score is computed outside the network", |
| "eligibility": "ELIGIBLE", |
| "priority": "P0", |
| "random_seed": 20260806 |
| }, |
| "artifacts": { |
| "fp32": { |
| "artifact_id": "AD01-mlcommons-4addd0f-fp32-tflite", |
| "source_url": "https://raw.githubusercontent.com/mlcommons/tiny/4addd0fa08d216e20637637874e084895f289da4/benchmark/training/anomaly_detection/trained_models/ad01_fp32.tflite", |
| "local_path": "{model_dir}/source/fp32/ad01_fp32.tflite", |
| "format": "TFLite", |
| "version": "git:4addd0fa08d216e20637637874e084895f289da4", |
| "sha256": "c66636f4d7f8af8b10518e7be750a22c9d8d46ec97326b40b0d94c097e0aad9b", |
| "license": "MIT" |
| }, |
| "public_quantized": { |
| "artifact_id": "AD01-mlcommons-4addd0f-int8-tflite", |
| "source_url": "https://raw.githubusercontent.com/mlcommons/tiny/4addd0fa08d216e20637637874e084895f289da4/benchmark/training/anomaly_detection/trained_models/ad01_int8.tflite", |
| "local_path": "{model_dir}/source/quantized/ad01_int8.tflite", |
| "format": "TFLite", |
| "version": "git:4addd0fa08d216e20637637874e084895f289da4", |
| "sha256": "87cf24194ef93d1d9b11a591d805526b98008e351655d29883c825c9c106ba24", |
| "license": "MIT" |
| } |
| }, |
| "stages": [ |
| { |
| "id": "acquire_fp32", |
| "stage": "source_acquisition", |
| "variant": "fp32", |
| "command": [ |
| "{python}", |
| "{repo_root}/scripts/stages/fetch_artifact.py", |
| "--url", |
| "https://raw.githubusercontent.com/mlcommons/tiny/4addd0fa08d216e20637637874e084895f289da4/benchmark/training/anomaly_detection/trained_models/ad01_fp32.tflite", |
| "--output", |
| "{model_dir}/source/fp32/ad01_fp32.tflite", |
| "--sha256", |
| "c66636f4d7f8af8b10518e7be750a22c9d8d46ec97326b40b0d94c097e0aad9b", |
| "--metadata", |
| "{model_dir}/source/fp32/download_metadata.json" |
| ], |
| "inputs": [], |
| "outputs": [ |
| "{model_dir}/source/fp32/ad01_fp32.tflite", |
| "{model_dir}/source/fp32/download_metadata.json" |
| ], |
| "requires": [], |
| "timeout_sec": 180, |
| "failure_code_on_error": "FAIL_SOURCE", |
| "validation_report": "{model_dir}/source/fp32/download_metadata.json", |
| "options": { |
| "immutable_download": true, |
| "tls": "1.2+" |
| }, |
| "patch": null |
| }, |
| { |
| "id": "validate_fp32", |
| "stage": "baseline_inference", |
| "variant": "fp32", |
| "command": [ |
| "{python}", |
| "{repo_root}/scripts/stages/inspect_tflite.py", |
| "--model", |
| "{model_dir}/source/fp32/ad01_fp32.tflite", |
| "--report", |
| "{model_dir}/baseline/fp32_validation.json", |
| "--seed", |
| "20260806", |
| "--threads", |
| "1" |
| ], |
| "inputs": [ |
| "{model_dir}/source/fp32/ad01_fp32.tflite" |
| ], |
| "outputs": [ |
| "{model_dir}/baseline/fp32_validation.json" |
| ], |
| "requires": [ |
| "acquire_fp32" |
| ], |
| "timeout_sec": 120, |
| "failure_code_on_error": "FAIL_BASELINE", |
| "validation_report": "{model_dir}/baseline/fp32_validation.json", |
| "options": { |
| "runtime": "ai-edge-litert", |
| "threads": 1, |
| "batch_size": 1 |
| }, |
| "patch": null |
| }, |
| { |
| "id": "acquire_quantized", |
| "stage": "public_quantized_acquisition", |
| "variant": "public_quantized", |
| "command": [ |
| "{python}", |
| "{repo_root}/scripts/stages/fetch_artifact.py", |
| "--url", |
| "https://raw.githubusercontent.com/mlcommons/tiny/4addd0fa08d216e20637637874e084895f289da4/benchmark/training/anomaly_detection/trained_models/ad01_int8.tflite", |
| "--output", |
| "{model_dir}/source/quantized/ad01_int8.tflite", |
| "--sha256", |
| "87cf24194ef93d1d9b11a591d805526b98008e351655d29883c825c9c106ba24", |
| "--metadata", |
| "{model_dir}/source/quantized/download_metadata.json" |
| ], |
| "inputs": [], |
| "outputs": [ |
| "{model_dir}/source/quantized/ad01_int8.tflite", |
| "{model_dir}/source/quantized/download_metadata.json" |
| ], |
| "requires": [], |
| "timeout_sec": 180, |
| "failure_code_on_error": "FAIL_PUBLIC_QUANTIZED_ARTIFACT", |
| "validation_report": "{model_dir}/source/quantized/download_metadata.json", |
| "options": { |
| "immutable_download": true, |
| "tls": "1.2+" |
| }, |
| "patch": null |
| }, |
| { |
| "id": "validate_quantized", |
| "stage": "public_quantized_inference", |
| "variant": "public_quantized", |
| "command": [ |
| "{python}", |
| "{repo_root}/scripts/stages/inspect_tflite.py", |
| "--model", |
| "{model_dir}/source/quantized/ad01_int8.tflite", |
| "--report", |
| "{model_dir}/baseline/public_quantized_validation.json", |
| "--seed", |
| "20260806", |
| "--threads", |
| "1" |
| ], |
| "inputs": [ |
| "{model_dir}/source/quantized/ad01_int8.tflite" |
| ], |
| "outputs": [ |
| "{model_dir}/baseline/public_quantized_validation.json" |
| ], |
| "requires": [ |
| "acquire_quantized" |
| ], |
| "timeout_sec": 120, |
| "failure_code_on_error": "FAIL_PUBLIC_QUANTIZED_LOAD", |
| "validation_report": "{model_dir}/baseline/public_quantized_validation.json", |
| "options": { |
| "runtime": "ai-edge-litert", |
| "threads": 1, |
| "batch_size": 1 |
| }, |
| "patch": null |
| }, |
| { |
| "id": "validate_pair", |
| "stage": "baseline_pair_compatibility", |
| "variant": "pair", |
| "artifact_id": "AD01-mlcommons-4addd0f-pair", |
| "command": [ |
| "{python}", |
| "{repo_root}/scripts/stages/compare_tflite_pair.py", |
| "--fp32-report", |
| "{model_dir}/baseline/fp32_validation.json", |
| "--quantized-report", |
| "{model_dir}/baseline/public_quantized_validation.json", |
| "--report", |
| "{model_dir}/baseline/pair_compatibility.json", |
| "--preprocessing-evidence", |
| "same upstream MLPerf Tiny DCASE ToyCar feature pipeline", |
| "--label-space-evidence", |
| "same 640-element reconstruction target and external anomaly scoring", |
| "--dataset-evidence", |
| "same DCASE 2020 Task 2 ToyCar trained-model release" |
| ], |
| "inputs": [ |
| "{model_dir}/baseline/fp32_validation.json", |
| "{model_dir}/baseline/public_quantized_validation.json" |
| ], |
| "outputs": [ |
| "{model_dir}/baseline/pair_compatibility.json" |
| ], |
| "requires": [ |
| "validate_fp32", |
| "validate_quantized" |
| ], |
| "timeout_sec": 60, |
| "failure_code_on_error": "FAIL_BASELINE_PAIR_MISMATCH", |
| "validation_report": "{model_dir}/baseline/pair_compatibility.json", |
| "options": { |
| "compare_operator_sequence": true, |
| "compare_io_shapes": true |
| }, |
| "patch": null |
| }, |
| { |
| "id": "convert_fp32_onnx", |
| "stage": "tflite_to_onnx_conversion", |
| "variant": "fp32", |
| "command": [ |
| "{python}", |
| "{repo_root}/scripts/stages/convert_tflite_to_onnx.py", |
| "--converter-python", |
| "{repo_root}/environment/converters/tf2onnx/.venv/bin/python", |
| "--input", |
| "{model_dir}/source/fp32/ad01_fp32.tflite", |
| "--output", |
| "{model_dir}/onnx/fp32/model_fp32.onnx", |
| "--report", |
| "{model_dir}/onnx/fp32/onnx_conversion.json", |
| "--tool-versions", |
| "{repo_root}/environment/converters/tf2onnx/tool_versions.json", |
| "--opset", |
| "16" |
| ], |
| "inputs": [ |
| "{model_dir}/source/fp32/ad01_fp32.tflite", |
| "{repo_root}/environment/converters/tf2onnx/tool_versions.json" |
| ], |
| "outputs": [ |
| "{model_dir}/onnx/fp32/model_fp32.onnx", |
| "{model_dir}/onnx/fp32/onnx_conversion.json" |
| ], |
| "requires": [ |
| "validate_fp32" |
| ], |
| "timeout_sec": 300, |
| "failure_code_on_error": "FAIL_EXPORT", |
| "validation_report": "{model_dir}/onnx/fp32/onnx_conversion.json", |
| "options": { |
| "converter": "tf2onnx 1.17.0", |
| "tensorflow": "2.20.0", |
| "opset": 16, |
| "dequantize": false, |
| "dynamic_axes": false |
| }, |
| "patch": null |
| }, |
| { |
| "id": "validate_fp32_onnx", |
| "stage": "onnx_checker_shape_inference_runtime", |
| "variant": "fp32", |
| "command": [ |
| "{python}", |
| "{repo_root}/scripts/stages/inspect_onnx.py", |
| "--model", |
| "{model_dir}/onnx/fp32/model_fp32.onnx", |
| "--report", |
| "{model_dir}/onnx/fp32/onnx_check.json", |
| "--inferred-model", |
| "{model_dir}/onnx/fp32/model_fp32.inferred.onnx", |
| "--seed", |
| "20260806", |
| "--threads", |
| "1" |
| ], |
| "inputs": [ |
| "{model_dir}/onnx/fp32/model_fp32.onnx" |
| ], |
| "outputs": [ |
| "{model_dir}/onnx/fp32/onnx_check.json", |
| "{model_dir}/onnx/fp32/model_fp32.inferred.onnx" |
| ], |
| "requires": [ |
| "convert_fp32_onnx" |
| ], |
| "timeout_sec": 120, |
| "failure_code_on_error": "FAIL_RUNTIME", |
| "validation_report": "{model_dir}/onnx/fp32/onnx_check.json", |
| "options": { |
| "checker": true, |
| "shape_inference": true, |
| "runtime": "onnxruntime 1.22.1", |
| "threads": 1 |
| }, |
| "patch": null |
| }, |
| { |
| "id": "compare_fp32_onnx", |
| "stage": "tflite_onnx_numerical_comparison", |
| "variant": "fp32", |
| "command": [ |
| "{python}", |
| "{repo_root}/scripts/stages/compare_tflite_onnx.py", |
| "--tflite", |
| "{model_dir}/source/fp32/ad01_fp32.tflite", |
| "--onnx", |
| "{model_dir}/onnx/fp32/model_fp32.onnx", |
| "--report", |
| "{model_dir}/onnx/fp32/onnx_output_compare.json", |
| "--seed", |
| "20260806", |
| "--threads", |
| "1", |
| "--atol", |
| "0.00001", |
| "--rtol", |
| "0.00001" |
| ], |
| "inputs": [ |
| "{model_dir}/source/fp32/ad01_fp32.tflite", |
| "{model_dir}/onnx/fp32/model_fp32.onnx" |
| ], |
| "outputs": [ |
| "{model_dir}/onnx/fp32/onnx_output_compare.json" |
| ], |
| "requires": [ |
| "validate_fp32_onnx" |
| ], |
| "timeout_sec": 120, |
| "failure_code_on_error": "FAIL_NUMERICAL_MISMATCH", |
| "validation_report": "{model_dir}/onnx/fp32/onnx_output_compare.json", |
| "options": { |
| "seed": 20260806, |
| "threads": 1, |
| "atol": 1e-05, |
| "rtol": 1e-05 |
| }, |
| "patch": null |
| }, |
| { |
| "id": "convert_quantized_onnx", |
| "stage": "tflite_to_onnx_conversion", |
| "variant": "public_quantized", |
| "command": [ |
| "{python}", |
| "{repo_root}/scripts/stages/convert_tflite_to_onnx.py", |
| "--converter-python", |
| "{repo_root}/environment/converters/tf2onnx/.venv/bin/python", |
| "--input", |
| "{model_dir}/source/quantized/ad01_int8.tflite", |
| "--output", |
| "{model_dir}/onnx/quantized/model_quantized.onnx", |
| "--report", |
| "{model_dir}/onnx/quantized/onnx_conversion.json", |
| "--tool-versions", |
| "{repo_root}/environment/converters/tf2onnx/tool_versions.json", |
| "--opset", |
| "16" |
| ], |
| "inputs": [ |
| "{model_dir}/source/quantized/ad01_int8.tflite", |
| "{repo_root}/environment/converters/tf2onnx/tool_versions.json" |
| ], |
| "outputs": [ |
| "{model_dir}/onnx/quantized/model_quantized.onnx", |
| "{model_dir}/onnx/quantized/onnx_conversion.json" |
| ], |
| "requires": [ |
| "validate_quantized" |
| ], |
| "timeout_sec": 300, |
| "failure_code_on_error": "FAIL_EXPORT", |
| "validation_report": "{model_dir}/onnx/quantized/onnx_conversion.json", |
| "options": { |
| "converter": "tf2onnx 1.17.0", |
| "tensorflow": "2.20.0", |
| "opset": 16, |
| "dequantize": false, |
| "dynamic_axes": false |
| }, |
| "patch": null |
| }, |
| { |
| "id": "validate_quantized_onnx", |
| "stage": "onnx_checker_shape_inference_runtime", |
| "variant": "public_quantized", |
| "command": [ |
| "{python}", |
| "{repo_root}/scripts/stages/inspect_onnx.py", |
| "--model", |
| "{model_dir}/onnx/quantized/model_quantized.onnx", |
| "--report", |
| "{model_dir}/onnx/quantized/onnx_check.json", |
| "--inferred-model", |
| "{model_dir}/onnx/quantized/model_quantized.inferred.onnx", |
| "--seed", |
| "20260806", |
| "--threads", |
| "1" |
| ], |
| "inputs": [ |
| "{model_dir}/onnx/quantized/model_quantized.onnx" |
| ], |
| "outputs": [ |
| "{model_dir}/onnx/quantized/onnx_check.json", |
| "{model_dir}/onnx/quantized/model_quantized.inferred.onnx" |
| ], |
| "requires": [ |
| "convert_quantized_onnx" |
| ], |
| "timeout_sec": 120, |
| "failure_code_on_error": "FAIL_RUNTIME", |
| "validation_report": "{model_dir}/onnx/quantized/onnx_check.json", |
| "options": { |
| "checker": true, |
| "shape_inference": true, |
| "runtime": "onnxruntime 1.22.1", |
| "threads": 1 |
| }, |
| "patch": null |
| }, |
| { |
| "id": "compare_quantized_onnx", |
| "stage": "tflite_onnx_numerical_comparison", |
| "variant": "public_quantized", |
| "command": [ |
| "{python}", |
| "{repo_root}/scripts/stages/compare_tflite_onnx.py", |
| "--tflite", |
| "{model_dir}/source/quantized/ad01_int8.tflite", |
| "--onnx", |
| "{model_dir}/onnx/quantized/model_quantized.onnx", |
| "--report", |
| "{model_dir}/onnx/quantized/onnx_output_compare.json", |
| "--seed", |
| "20260806", |
| "--threads", |
| "1", |
| "--atol", |
| "0", |
| "--rtol", |
| "0" |
| ], |
| "inputs": [ |
| "{model_dir}/source/quantized/ad01_int8.tflite", |
| "{model_dir}/onnx/quantized/model_quantized.onnx" |
| ], |
| "outputs": [ |
| "{model_dir}/onnx/quantized/onnx_output_compare.json" |
| ], |
| "requires": [ |
| "validate_quantized_onnx" |
| ], |
| "timeout_sec": 120, |
| "failure_code_on_error": "FAIL_NUMERICAL_MISMATCH", |
| "validation_report": "{model_dir}/onnx/quantized/onnx_output_compare.json", |
| "options": { |
| "seed": 20260806, |
| "threads": 1, |
| "atol": 0.0, |
| "rtol": 0.0, |
| "comparison": "raw_integer" |
| }, |
| "patch": null |
| }, |
| { |
| "id": "inspect_quantized_onnx_preservation", |
| "stage": "onnx_quantization_preservation", |
| "variant": "public_quantized", |
| "command": [ |
| "{python}", |
| "{repo_root}/scripts/stages/inspect_onnx_quantization.py", |
| "--onnx-check", |
| "{model_dir}/onnx/quantized/onnx_check.json", |
| "--report", |
| "{model_dir}/onnx/quantized/onnx_quantization_inspection.json" |
| ], |
| "inputs": [ |
| "{model_dir}/onnx/quantized/onnx_check.json" |
| ], |
| "outputs": [ |
| "{model_dir}/onnx/quantized/onnx_quantization_inspection.json" |
| ], |
| "requires": [ |
| "validate_quantized_onnx" |
| ], |
| "timeout_sec": 60, |
| "failure_code_on_error": "FAIL_QUANTIZATION_PRESERVATION", |
| "validation_report": "{model_dir}/onnx/quantized/onnx_quantization_inspection.json", |
| "options": { |
| "allowed_representations": [ |
| "QDQ", |
| "QOPERATOR", |
| "QUANTIZED_TENSOR" |
| ], |
| "float_fallback_is_failure": true |
| }, |
| "patch": null |
| } |
| ] |
| } |
|
|
