| { |
| "schema_version": "1.0", |
| "model_id": "VC02", |
| "evaluation_name": "mlperf_tiny_ic01_official_subset_quality", |
| "random_seed": 20260806, |
| "dataset": { |
| "name": "cifar10_python_archive", |
| "title": "CIFAR-10 Python version", |
| "version": "CIFAR-10 Python archive published 2009-06-04", |
| "license": "No explicit dataset license on the official download page", |
| "license_note": "The official CIFAR-10 page requests citation of the Learning Multiple Layers of Features from Tiny Images technical report but does not state an explicit dataset license or require credentials/click-through.", |
| "canonical_url": "https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz", |
| "url": "https://huggingface.co/Peyiloo/peyiloo/resolve/main/cifar-10-python.tar.gz", |
| "transport_note": "The canonical Toronto server timed out repeatedly on 2026-08-06. This transport mirror is accepted only because its Git LFS pointer SHA-256/size and the downloaded bytes exactly match the SHA-256/size published by Keras for the canonical Toronto URL; the canonical source provenance is unchanged.", |
| "path": "research/downloads/cifar10/cifar-10-python.tar.gz", |
| "expected_bytes": 170498071, |
| "expected_sha256": "6d958be074577803d12ecdefd02955f39262c83c16fe9348329d7fe0b5c001ce", |
| "expected_md5": "c58f30108f718f92721af3b95e74349a", |
| "use": "evaluation_only_no_training_or_calibration" |
| }, |
| "upstream": { |
| "repository": "https://github.com/mlcommons/tiny", |
| "commit": "4addd0fa08d216e20637637874e084895f289da4", |
| "files": [ |
| { |
| "name": "datasets_README.md", |
| "url": "https://raw.githubusercontent.com/mlcommons/tiny/4addd0fa08d216e20637637874e084895f289da4/benchmark/evaluation/datasets/README.md", |
| "path": "models/vision_classification/VC02/quality_evaluation/sources/datasets_README.md", |
| "expected_sha256": "ed00f61828db845481209d1b3f320f5718085b5ff39ecb6a229fa5314f320ca4", |
| "license": "Apache-2.0" |
| }, |
| { |
| "name": "y_labels.csv", |
| "url": "https://raw.githubusercontent.com/mlcommons/tiny/4addd0fa08d216e20637637874e084895f289da4/benchmark/evaluation/datasets/ic01/y_labels.csv", |
| "path": "models/vision_classification/VC02/quality_evaluation/sources/y_labels.csv", |
| "expected_sha256": "79e1f19ad731f046983a1a4480bda6369df26c5acee47505a860f164fb58ccd4", |
| "license": "Apache-2.0" |
| }, |
| { |
| "name": "perf_samples_idxs.npy", |
| "url": "https://raw.githubusercontent.com/mlcommons/tiny/4addd0fa08d216e20637637874e084895f289da4/benchmark/training/image_classification/perf_samples_idxs.npy", |
| "path": "models/vision_classification/VC02/quality_evaluation/sources/perf_samples_idxs.npy", |
| "expected_sha256": "3bd4a88eeb4c50fad652d0f24c8af13bc9219ba2878aea47c6536bfbeb43024d", |
| "license": "Apache-2.0" |
| }, |
| { |
| "name": "tflite_test.py", |
| "url": "https://raw.githubusercontent.com/mlcommons/tiny/4addd0fa08d216e20637637874e084895f289da4/benchmark/training/image_classification/tflite_test.py", |
| "path": "models/vision_classification/VC02/quality_evaluation/sources/tflite_test.py", |
| "expected_sha256": "9982e3225a9b4baf55aaee8fcdf1128ecde0ae57745ba9ce3be5c0f9f56ac47a", |
| "license": "Apache-2.0" |
| }, |
| { |
| "name": "train.py", |
| "url": "https://raw.githubusercontent.com/mlcommons/tiny/4addd0fa08d216e20637637874e084895f289da4/benchmark/training/image_classification/train.py", |
| "path": "models/vision_classification/VC02/quality_evaluation/sources/train.py", |
| "expected_sha256": "4c238829dc8f89961fb136d184565693350dc17ed9b3466b279a564f02d9c592", |
| "license": "Apache-2.0" |
| }, |
| { |
| "name": "LICENSE.md", |
| "url": "https://raw.githubusercontent.com/mlcommons/tiny/4addd0fa08d216e20637637874e084895f289da4/LICENSE.md", |
| "path": "models/vision_classification/VC02/quality_evaluation/sources/LICENSE.md", |
| "expected_sha256": "0d542e0c8804e39aa7f37eb00da5a762149dc682d7829451287e11b938e94594", |
| "license": "Apache-2.0" |
| }, |
| { |
| "name": "keras_v3.12.0_cifar10.py", |
| "url": "https://raw.githubusercontent.com/keras-team/keras/v3.12.0/keras/src/datasets/cifar10.py", |
| "path": "models/vision_classification/VC02/quality_evaluation/sources/keras_v3.12.0_cifar10.py", |
| "expected_sha256": "bf651dbec372cef81ab162bda19d1a8476b513a3f4976b40240616943a45736f", |
| "license": "Apache-2.0" |
| }, |
| { |
| "name": "transport_mirror_lfs_pointer.txt", |
| "url": "https://huggingface.co/Peyiloo/peyiloo/raw/main/cifar-10-python.tar.gz", |
| "path": "models/vision_classification/VC02/quality_evaluation/sources/transport_mirror_lfs_pointer.txt", |
| "expected_sha256": "3684b5b4c4935ed2a1e722ac45675d619fe1f16930be5439803edf03047605eb", |
| "license": "Provenance pointer only; dataset terms remain those of the canonical source" |
| } |
| ] |
| }, |
| "models": { |
| "fp32": { |
| "path": "models/vision_classification/VC02/source/fp32/source_fp32.tflite", |
| "source_url": "https://raw.githubusercontent.com/mlcommons/tiny/4addd0fa08d216e20637637874e084895f289da4/benchmark/training/image_classification/trained_models/pretrainedResnet.tflite", |
| "expected_sha256": "b5c0046d6e0328b4956afd6baa29555a29b1f1c65bdd45aaed75b7cd484d9f79" |
| }, |
| "public_int8": { |
| "path": "models/vision_classification/VC02/source/quantized/source_public_quantized.tflite", |
| "source_url": "https://raw.githubusercontent.com/mlcommons/tiny/4addd0fa08d216e20637637874e084895f289da4/benchmark/training/image_classification/trained_models/pretrainedResnet_quant.tflite", |
| "expected_sha256": "3c002613d1b2475eb51dd78dfb85a546c8ae658dee71cf6ade43b022fe205415" |
| } |
| }, |
| "protocol": { |
| "benchmark_id": "ic01", |
| "dataset_loader": "cifar10_python_test_batch", |
| "labels_path": "models/vision_classification/VC02/quality_evaluation/sources/y_labels.csv", |
| "indices_path": "models/vision_classification/VC02/quality_evaluation/sources/perf_samples_idxs.npy", |
| "input_shape": [1, 32, 32, 3], |
| "fp32_input_mode": "raw_uint8_as_float32", |
| "public_int8_input": "subtract 128 from the same raw RGB bytes, equivalently artifact scale 1 and zero point -128", |
| "class_count": 10, |
| "label_space": ["airplane", "automobile", "bird", "cat", "deer", "dog", "frog", "horse", "ship", "truck"], |
| "label_counts": {"0": 20, "1": 20, "2": 20, "3": 20, "4": 20, "5": 20, "6": 20, "7": 20, "8": 20, "9": 20}, |
| "sample_order": "pinned perf_samples_idxs.npy and y_labels.csv order", |
| "metric_equation": "correct argmax predictions divided by 200 official stimuli" |
| }, |
| "metric": { |
| "name": "top1_accuracy", |
| "sample_count": 200, |
| "acceptance_threshold": 0.85, |
| "threshold_source": "models/vision_classification/VC02/quality_evaluation/sources/datasets_README.md" |
| }, |
| "runtime": { |
| "interpreter": "ai-edge-litert", |
| "threads": 1, |
| "batch_size": 1, |
| "quantized_input_rounding": "numpy_rint_then_int8_clip", |
| "output_decision": "argmax_raw_tensor" |
| }, |
| "provenance": { |
| "fetch_report": "models/vision_classification/VC02/quality_evaluation/fetch_report.json", |
| "stdout_log": "models/vision_classification/VC02/quality_evaluation/logs/evaluation.stdout.log", |
| "stderr_log": "models/vision_classification/VC02/quality_evaluation/logs/evaluation.stderr.log", |
| "validation_report": "models/vision_classification/VC02/quality_evaluation/results/validation_report.json" |
| } |
| } |
|
|
