| { |
| "schema_version": "1.0", |
| "model": { |
| "model_id": "SP08", |
| "model_name": "TensorFlow Lite Micro micro_speech tiny_conv", |
| "task": "keyword spotting", |
| "architecture_family": "tiny depthwise convolution plus fully connected", |
| "source_framework": "TensorFlow frozen GraphDef and TensorFlow Lite", |
| "source_repository": "https://android.googlesource.com/platform/external/tensorflow/+/4ed0453e335/tensorflow/lite/micro/examples/micro_speech/train/", |
| "license": "Apache-2.0; source LICENSE SHA-256 37dbed59a6aef0803f24a7dac146679491c460faa8bc3834974d786acf56a40f", |
| "public_quantized_available": true, |
| "paired_fp32_available": true, |
| "pair_compatibility": "VERIFIED", |
| "dataset": "Google Speech Commands v2", |
| "metric": "task-specific quality metric", |
| "input_shape": [ |
| 1, |
| 49, |
| 40, |
| 1 |
| ], |
| "output_shape": [ |
| 1, |
| 4 |
| ], |
| "preprocessing": "16 kHz mono; 1 s; microfrontend spectrogram; 30 ms feature slice; 20 ms stride; 49 slices; 40 bins", |
| "label_space": "ordered 4 classes: silence; unknown; yes; no", |
| "eligibility": "ELIGIBLE", |
| "priority": "P0", |
| "random_seed": 20260806 |
| }, |
| "artifacts": { |
| "fp32": { |
| "artifact_id": "SP08-registered-fp32-graphdef", |
| "source_url": "https://storage.googleapis.com/download.tensorflow.org/models/tflite/micro/micro_speech_2020_04_13.zip#micro_speech/models/model.pb", |
| "local_path": "{model_dir}/source/fp32/source_fp32.pb", |
| "format": "TensorFlow GraphDef/SavedModel", |
| "version": "micro_speech_2020_04_13 release archive; TensorFlow source snapshot 4ed0453e335", |
| "sha256": "61f3f6db6a0eb11293c8b64d9916da9095ad30d9042db2227b86824d29dfec4e", |
| "license": "Apache-2.0; source LICENSE SHA-256 37dbed59a6aef0803f24a7dac146679491c460faa8bc3834974d786acf56a40f" |
| }, |
| "public_quantized": { |
| "artifact_id": "SP08-public-quantized-01", |
| "source_url": "https://storage.googleapis.com/download.tensorflow.org/models/tflite/micro/micro_speech_2020_04_13.zip#micro_speech/models/model.tflite", |
| "local_path": "{model_dir}/source/quantized/source_public_quantized.tflite", |
| "format": "TFLite FlatBuffer inside ZIP", |
| "version": "micro_speech_2020_04_13 release archive; TensorFlow source snapshot 4ed0453e335", |
| "sha256": "454779dcfea05290759256178162fa86eb17642e7c9cac0de8ea78e1693cee00", |
| "license": "Apache-2.0; source LICENSE SHA-256 37dbed59a6aef0803f24a7dac146679491c460faa8bc3834974d786acf56a40f" |
| } |
| }, |
| "stages": [ |
| { |
| "id": "acquire_fp32", |
| "stage": "source_acquisition", |
| "variant": "fp32", |
| "command": [ |
| "{python}", |
| "{repo_root}/scripts/stages/copy_artifact.py", |
| "--input", |
| "{repo_root}/research/downloads/speech_kws/micro_speech_2020_04_13/micro_speech/models/model.pb", |
| "--output", |
| "{model_dir}/source/fp32/source_fp32.pb", |
| "--sha256", |
| "61f3f6db6a0eb11293c8b64d9916da9095ad30d9042db2227b86824d29dfec4e", |
| "--report", |
| "{model_dir}/source/fp32/copy_metadata.json" |
| ], |
| "inputs": [ |
| "{repo_root}/research/downloads/speech_kws/micro_speech_2020_04_13/micro_speech/models/model.pb" |
| ], |
| "outputs": [ |
| "{model_dir}/source/fp32/source_fp32.pb", |
| "{model_dir}/source/fp32/copy_metadata.json" |
| ], |
| "requires": [], |
| "timeout_sec": 180, |
| "failure_code_on_error": "FAIL_SOURCE", |
| "validation_report": "{model_dir}/source/fp32/copy_metadata.json", |
| "options": { |
| "route": "IMMUTABLE_LOCAL_COPY_FROM_REGISTRY", |
| "checksum_required": true, |
| "quantization_generated": false, |
| "weights_modified": false |
| }, |
| "patch": null |
| }, |
| { |
| "id": "validate_fp32", |
| "stage": "baseline_tensorflow_inference", |
| "variant": "fp32", |
| "command": [ |
| "{repo_root}/environment/converters/tf2onnx/.venv/bin/python", |
| "{repo_root}/scripts/stages/inspect_tensorflow.py", |
| "--format", |
| "graphdef", |
| "--model", |
| "{model_dir}/source/fp32/source_fp32.pb", |
| "--report", |
| "{model_dir}/baseline/fp32_validation.json", |
| "--input-shape", |
| "1,49,40,1", |
| "--input-min", |
| "0.0", |
| "--input-max", |
| "26.0", |
| "--seed", |
| "20260806", |
| "--threads", |
| "1", |
| "--input-name", |
| "wav_data:0", |
| "--output-name", |
| "labels_softmax:0", |
| "--input-mode", |
| "silence_wav" |
| ], |
| "inputs": [ |
| "{model_dir}/source/fp32/source_fp32.pb" |
| ], |
| "outputs": [ |
| "{model_dir}/baseline/fp32_validation.json" |
| ], |
| "requires": [ |
| "acquire_fp32" |
| ], |
| "timeout_sec": 600, |
| "failure_code_on_error": "FAIL_BASELINE", |
| "validation_report": "{model_dir}/baseline/fp32_validation.json", |
| "options": { |
| "runtime": "tensorflow-cpu 2.20.0", |
| "threads": 1, |
| "batch_size": 1 |
| }, |
| "patch": null |
| }, |
| { |
| "id": "acquire_quantized", |
| "stage": "public_quantized_acquisition", |
| "variant": "public_quantized", |
| "command": [ |
| "{python}", |
| "{repo_root}/scripts/stages/copy_artifact.py", |
| "--input", |
| "{repo_root}/research/downloads/speech_kws/micro_speech_2020_04_13/micro_speech/models/model.tflite", |
| "--output", |
| "{model_dir}/source/quantized/source_public_quantized.tflite", |
| "--sha256", |
| "454779dcfea05290759256178162fa86eb17642e7c9cac0de8ea78e1693cee00", |
| "--report", |
| "{model_dir}/source/quantized/copy_metadata.json" |
| ], |
| "inputs": [ |
| "{repo_root}/research/downloads/speech_kws/micro_speech_2020_04_13/micro_speech/models/model.tflite" |
| ], |
| "outputs": [ |
| "{model_dir}/source/quantized/source_public_quantized.tflite", |
| "{model_dir}/source/quantized/copy_metadata.json" |
| ], |
| "requires": [], |
| "timeout_sec": 180, |
| "failure_code_on_error": "FAIL_PUBLIC_QUANTIZED_ARTIFACT", |
| "validation_report": "{model_dir}/source/quantized/copy_metadata.json", |
| "options": { |
| "route": "IMMUTABLE_LOCAL_COPY_FROM_REGISTRY", |
| "checksum_required": true, |
| "quantization_generated": false, |
| "weights_modified": false |
| }, |
| "patch": null |
| }, |
| { |
| "id": "validate_quantized", |
| "stage": "public_quantized_inference", |
| "variant": "public_quantized", |
| "command": [ |
| "{python}", |
| "{repo_root}/scripts/stages/inspect_tflite.py", |
| "--model", |
| "{model_dir}/source/quantized/source_public_quantized.tflite", |
| "--report", |
| "{model_dir}/baseline/public_quantized_validation.json", |
| "--seed", |
| "20260806", |
| "--threads", |
| "1" |
| ], |
| "inputs": [ |
| "{model_dir}/source/quantized/source_public_quantized.tflite" |
| ], |
| "outputs": [ |
| "{model_dir}/baseline/public_quantized_validation.json" |
| ], |
| "requires": [ |
| "acquire_quantized" |
| ], |
| "timeout_sec": 600, |
| "failure_code_on_error": "FAIL_PUBLIC_QUANTIZED_LOAD", |
| "validation_report": "{model_dir}/baseline/public_quantized_validation.json", |
| "options": { |
| "runtime": "ai-edge-litert 2.1.6", |
| "threads": 1, |
| "batch_size": 1 |
| }, |
| "patch": null |
| }, |
| { |
| "id": "document_pair_boundary", |
| "stage": "mixed_format_pair_boundary_documentation", |
| "variant": "pair", |
| "artifact_id": "SP08-mixed-format-pair-boundary", |
| "command": [ |
| "{python}", |
| "{repo_root}/scripts/stages/document_mixed_pair_boundary.py", |
| "--fp32-report", |
| "{model_dir}/baseline/fp32_validation.json", |
| "--quantized-report", |
| "{model_dir}/baseline/public_quantized_validation.json", |
| "--report", |
| "{model_dir}/baseline/mixed_pair_boundary.json", |
| "--boundary", |
| "wrapper_to_core", |
| "--preprocessing-evidence", |
| "16 kHz mono; 1 s; microfrontend spectrogram; 30 ms feature slice; 20 ms stride; 49 slices; 40 bins", |
| "--label-space-evidence", |
| "ordered 4 classes: silence; unknown; yes; no", |
| "--dataset-evidence", |
| "Google Speech Commands v2", |
| "--core-input-shape", |
| "1,49,40,1" |
| ], |
| "inputs": [ |
| "{model_dir}/baseline/fp32_validation.json", |
| "{model_dir}/baseline/public_quantized_validation.json" |
| ], |
| "outputs": [ |
| "{model_dir}/baseline/mixed_pair_boundary.json" |
| ], |
| "requires": [ |
| "validate_fp32", |
| "validate_quantized" |
| ], |
| "timeout_sec": 120, |
| "failure_code_on_error": "FAIL_BASELINE_PAIR_MISMATCH", |
| "validation_report": "{model_dir}/baseline/mixed_pair_boundary.json", |
| "options": { |
| "boundary": "wrapper_to_core", |
| "does_not_replace_registry_pair_evidence": true |
| }, |
| "patch": null |
| }, |
| { |
| "id": "convert_fp32_onnx", |
| "stage": "tensorflow_fp32_to_onnx_conversion", |
| "variant": "fp32", |
| "command": [ |
| "{repo_root}/environment/converters/tf2onnx/.venv/bin/python", |
| "{repo_root}/scripts/stages/convert_tensorflow_to_onnx.py", |
| "--input", |
| "{model_dir}/source/fp32/source_fp32.pb", |
| "--output", |
| "{model_dir}/onnx/fp32/model_fp32.onnx", |
| "--report", |
| "{model_dir}/onnx/fp32/onnx_conversion.json", |
| "--opset", |
| "16", |
| "--threads", |
| "1", |
| "--format", |
| "graphdef", |
| "--input-shape", |
| "1,49,40,1", |
| "--input-name", |
| "Reshape_2:0", |
| "--output-name", |
| "labels_softmax:0" |
| ], |
| "inputs": [ |
| "{model_dir}/source/fp32/source_fp32.pb", |
| "{repo_root}/environment/converters/tf2onnx/tool_versions.json" |
| ], |
| "outputs": [ |
| "{model_dir}/onnx/fp32/model_fp32.onnx", |
| "{model_dir}/onnx/fp32/onnx_conversion.json" |
| ], |
| "requires": [ |
| "acquire_fp32" |
| ], |
| "timeout_sec": 1200, |
| "failure_code_on_error": "FAIL_EXPORT", |
| "validation_report": "{model_dir}/onnx/fp32/onnx_conversion.json", |
| "options": { |
| "converter": "tf2onnx 1.17.0", |
| "tensorflow": "2.20.0", |
| "opset": 16, |
| "inference_only": true, |
| "fixed_shape": [ |
| 1, |
| 49, |
| 40, |
| 1 |
| ], |
| "quantization_generated": false |
| }, |
| "patch": "INFERENCE_BOUNDARY_ONLY: cut the immutable GraphDef at existing Reshape_2:0 core input and fix shape 1x49x40x1; audio frontend remains external; no layer or weight change" |
| }, |
| { |
| "id": "validate_fp32_onnx", |
| "stage": "onnx_checker_shape_inference_runtime", |
| "variant": "fp32", |
| "command": [ |
| "{python}", |
| "{repo_root}/scripts/stages/inspect_onnx.py", |
| "--model", |
| "{model_dir}/onnx/fp32/model_fp32.onnx", |
| "--report", |
| "{model_dir}/onnx/fp32/onnx_check.json", |
| "--inferred-model", |
| "{model_dir}/onnx/fp32/model_fp32.inferred.onnx", |
| "--seed", |
| "20260806", |
| "--threads", |
| "1" |
| ], |
| "inputs": [ |
| "{model_dir}/onnx/fp32/model_fp32.onnx" |
| ], |
| "outputs": [ |
| "{model_dir}/onnx/fp32/onnx_check.json", |
| "{model_dir}/onnx/fp32/model_fp32.inferred.onnx" |
| ], |
| "requires": [ |
| "convert_fp32_onnx" |
| ], |
| "timeout_sec": 900, |
| "failure_code_on_error": "FAIL_RUNTIME", |
| "validation_report": "{model_dir}/onnx/fp32/onnx_check.json", |
| "options": { |
| "checker": true, |
| "shape_inference": true, |
| "runtime": "onnxruntime 1.22.1" |
| }, |
| "patch": null |
| }, |
| { |
| "id": "compare_fp32_onnx", |
| "stage": "tensorflow_onnx_numerical_comparison", |
| "variant": "fp32", |
| "command": [ |
| "{repo_root}/environment/converters/tf2onnx/.venv/bin/python", |
| "{repo_root}/scripts/stages/compare_tensorflow_target.py", |
| "--source-format", |
| "graphdef", |
| "--target-format", |
| "onnx", |
| "--source", |
| "{model_dir}/source/fp32/source_fp32.pb", |
| "--target", |
| "{model_dir}/onnx/fp32/model_fp32.onnx", |
| "--report", |
| "{model_dir}/onnx/fp32/onnx_output_compare.json", |
| "--input-shape", |
| "1,49,40,1", |
| "--input-min", |
| "0.0", |
| "--input-max", |
| "26.0", |
| "--seed", |
| "20260806", |
| "--threads", |
| "1", |
| "--atol", |
| "1e-05", |
| "--rtol", |
| "1e-05", |
| "--input-name", |
| "Reshape_2:0", |
| "--output-name", |
| "labels_softmax:0" |
| ], |
| "inputs": [ |
| "{model_dir}/source/fp32/source_fp32.pb", |
| "{model_dir}/onnx/fp32/model_fp32.onnx" |
| ], |
| "outputs": [ |
| "{model_dir}/onnx/fp32/onnx_output_compare.json" |
| ], |
| "requires": [ |
| "validate_fp32_onnx" |
| ], |
| "timeout_sec": 900, |
| "failure_code_on_error": "FAIL_NUMERICAL_MISMATCH", |
| "validation_report": "{model_dir}/onnx/fp32/onnx_output_compare.json", |
| "options": { |
| "same_seed_source_vs_target": true, |
| "atol": 1e-05, |
| "rtol": 1e-05, |
| "official_quality_metric": "NOT_RUN" |
| }, |
| "patch": "INFERENCE_BOUNDARY_ONLY: cut the immutable GraphDef at existing Reshape_2:0 core input and fix shape 1x49x40x1; audio frontend remains external; no layer or weight change" |
| }, |
| { |
| "id": "convert_quantized_onnx", |
| "stage": "public_quantized_tflite_to_onnx", |
| "variant": "public_quantized", |
| "command": [ |
| "{python}", |
| "{repo_root}/scripts/stages/convert_tflite_to_onnx.py", |
| "--converter-python", |
| "{repo_root}/environment/converters/tf2onnx/.venv/bin/python", |
| "--input", |
| "{model_dir}/source/quantized/source_public_quantized.tflite", |
| "--output", |
| "{model_dir}/onnx/quantized/model_quantized.onnx", |
| "--report", |
| "{model_dir}/onnx/quantized/onnx_conversion.json", |
| "--tool-versions", |
| "{repo_root}/environment/converters/tf2onnx/tool_versions.json", |
| "--opset", |
| "16" |
| ], |
| "inputs": [ |
| "{model_dir}/source/quantized/source_public_quantized.tflite", |
| "{repo_root}/environment/converters/tf2onnx/tool_versions.json" |
| ], |
| "outputs": [ |
| "{model_dir}/onnx/quantized/model_quantized.onnx", |
| "{model_dir}/onnx/quantized/onnx_conversion.json" |
| ], |
| "requires": [ |
| "acquire_quantized" |
| ], |
| "timeout_sec": 1200, |
| "failure_code_on_error": "FAIL_EXPORT", |
| "validation_report": "{model_dir}/onnx/quantized/onnx_conversion.json", |
| "options": { |
| "dequantize": false, |
| "quantization_generated": false, |
| "independent_variant_conversion": true |
| }, |
| "patch": null |
| }, |
| { |
| "id": "validate_quantized_onnx", |
| "stage": "onnx_checker_shape_inference_runtime", |
| "variant": "public_quantized", |
| "command": [ |
| "{python}", |
| "{repo_root}/scripts/stages/inspect_onnx.py", |
| "--model", |
| "{model_dir}/onnx/quantized/model_quantized.onnx", |
| "--report", |
| "{model_dir}/onnx/quantized/onnx_check.json", |
| "--inferred-model", |
| "{model_dir}/onnx/quantized/model_quantized.inferred.onnx", |
| "--seed", |
| "20260806", |
| "--threads", |
| "1" |
| ], |
| "inputs": [ |
| "{model_dir}/onnx/quantized/model_quantized.onnx" |
| ], |
| "outputs": [ |
| "{model_dir}/onnx/quantized/onnx_check.json", |
| "{model_dir}/onnx/quantized/model_quantized.inferred.onnx" |
| ], |
| "requires": [ |
| "convert_quantized_onnx" |
| ], |
| "timeout_sec": 900, |
| "failure_code_on_error": "FAIL_RUNTIME", |
| "validation_report": "{model_dir}/onnx/quantized/onnx_check.json", |
| "options": { |
| "checker": true, |
| "shape_inference": true, |
| "runtime": "onnxruntime 1.22.1" |
| }, |
| "patch": null |
| }, |
| { |
| "id": "inspect_quantized_onnx_preservation", |
| "stage": "onnx_quantization_preservation", |
| "variant": "public_quantized", |
| "command": [ |
| "{python}", |
| "{repo_root}/scripts/stages/inspect_onnx_quantization.py", |
| "--onnx-check", |
| "{model_dir}/onnx/quantized/onnx_check.json", |
| "--report", |
| "{model_dir}/onnx/quantized/onnx_quantization_inspection.json" |
| ], |
| "inputs": [ |
| "{model_dir}/onnx/quantized/onnx_check.json" |
| ], |
| "outputs": [ |
| "{model_dir}/onnx/quantized/onnx_quantization_inspection.json" |
| ], |
| "requires": [ |
| "validate_quantized_onnx" |
| ], |
| "timeout_sec": 120, |
| "failure_code_on_error": "FAIL_QUANTIZATION_PRESERVATION", |
| "validation_report": "{model_dir}/onnx/quantized/onnx_quantization_inspection.json", |
| "options": { |
| "float_fallback_is_failure": true, |
| "accepted": [ |
| "QDQ", |
| "QOPERATOR", |
| "MIXED_QOPERATOR_QDQ" |
| ] |
| }, |
| "patch": null |
| }, |
| { |
| "id": "compare_quantized_onnx", |
| "stage": "tflite_onnx_numerical_comparison", |
| "variant": "public_quantized", |
| "command": [ |
| "{python}", |
| "{repo_root}/scripts/stages/compare_tflite_onnx.py", |
| "--tflite", |
| "{model_dir}/source/quantized/source_public_quantized.tflite", |
| "--onnx", |
| "{model_dir}/onnx/quantized/model_quantized.onnx", |
| "--report", |
| "{model_dir}/onnx/quantized/onnx_output_compare.json", |
| "--seed", |
| "20260806", |
| "--threads", |
| "1", |
| "--atol", |
| "0", |
| "--rtol", |
| "0" |
| ], |
| "inputs": [ |
| "{model_dir}/source/quantized/source_public_quantized.tflite", |
| "{model_dir}/onnx/quantized/model_quantized.onnx" |
| ], |
| "outputs": [ |
| "{model_dir}/onnx/quantized/onnx_output_compare.json" |
| ], |
| "requires": [ |
| "validate_quantized_onnx" |
| ], |
| "timeout_sec": 900, |
| "failure_code_on_error": "FAIL_NUMERICAL_MISMATCH", |
| "validation_report": "{model_dir}/onnx/quantized/onnx_output_compare.json", |
| "options": { |
| "strict_integer_or_float_output_match": true, |
| "atol": 0.0, |
| "rtol": 0.0 |
| }, |
| "patch": null |
| } |
| ] |
| } |
|
|
