{
  "schema_version": "1.0",
  "model_id": "VC02",
  "evaluation_name": "mlperf_tiny_ic01_official_subset_quality",
  "random_seed": 20260806,
  "dataset": {
    "name": "cifar10_python_archive",
    "title": "CIFAR-10 Python version",
    "version": "CIFAR-10 Python archive published 2009-06-04",
    "license": "No explicit dataset license on the official download page",
    "license_note": "The official CIFAR-10 page requests citation of the Learning Multiple Layers of Features from Tiny Images technical report but does not state an explicit dataset license or require credentials/click-through.",
    "canonical_url": "https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz",
    "url": "https://huggingface.co/Peyiloo/peyiloo/resolve/main/cifar-10-python.tar.gz",
    "transport_note": "The canonical Toronto server timed out repeatedly on 2026-08-06. This transport mirror is accepted only because its Git LFS pointer SHA-256/size and the downloaded bytes exactly match the SHA-256/size published by Keras for the canonical Toronto URL; the canonical source provenance is unchanged.",
    "path": "research/downloads/cifar10/cifar-10-python.tar.gz",
    "expected_bytes": 170498071,
    "expected_sha256": "6d958be074577803d12ecdefd02955f39262c83c16fe9348329d7fe0b5c001ce",
    "expected_md5": "c58f30108f718f92721af3b95e74349a",
    "use": "evaluation_only_no_training_or_calibration"
  },
  "upstream": {
    "repository": "https://github.com/mlcommons/tiny",
    "commit": "4addd0fa08d216e20637637874e084895f289da4",
    "files": [
      {
        "name": "datasets_README.md",
        "url": "https://raw.githubusercontent.com/mlcommons/tiny/4addd0fa08d216e20637637874e084895f289da4/benchmark/evaluation/datasets/README.md",
        "path": "models/vision_classification/VC02/quality_evaluation/sources/datasets_README.md",
        "expected_sha256": "ed00f61828db845481209d1b3f320f5718085b5ff39ecb6a229fa5314f320ca4",
        "license": "Apache-2.0"
      },
      {
        "name": "y_labels.csv",
        "url": "https://raw.githubusercontent.com/mlcommons/tiny/4addd0fa08d216e20637637874e084895f289da4/benchmark/evaluation/datasets/ic01/y_labels.csv",
        "path": "models/vision_classification/VC02/quality_evaluation/sources/y_labels.csv",
        "expected_sha256": "79e1f19ad731f046983a1a4480bda6369df26c5acee47505a860f164fb58ccd4",
        "license": "Apache-2.0"
      },
      {
        "name": "perf_samples_idxs.npy",
        "url": "https://raw.githubusercontent.com/mlcommons/tiny/4addd0fa08d216e20637637874e084895f289da4/benchmark/training/image_classification/perf_samples_idxs.npy",
        "path": "models/vision_classification/VC02/quality_evaluation/sources/perf_samples_idxs.npy",
        "expected_sha256": "3bd4a88eeb4c50fad652d0f24c8af13bc9219ba2878aea47c6536bfbeb43024d",
        "license": "Apache-2.0"
      },
      {
        "name": "tflite_test.py",
        "url": "https://raw.githubusercontent.com/mlcommons/tiny/4addd0fa08d216e20637637874e084895f289da4/benchmark/training/image_classification/tflite_test.py",
        "path": "models/vision_classification/VC02/quality_evaluation/sources/tflite_test.py",
        "expected_sha256": "9982e3225a9b4baf55aaee8fcdf1128ecde0ae57745ba9ce3be5c0f9f56ac47a",
        "license": "Apache-2.0"
      },
      {
        "name": "train.py",
        "url": "https://raw.githubusercontent.com/mlcommons/tiny/4addd0fa08d216e20637637874e084895f289da4/benchmark/training/image_classification/train.py",
        "path": "models/vision_classification/VC02/quality_evaluation/sources/train.py",
        "expected_sha256": "4c238829dc8f89961fb136d184565693350dc17ed9b3466b279a564f02d9c592",
        "license": "Apache-2.0"
      },
      {
        "name": "LICENSE.md",
        "url": "https://raw.githubusercontent.com/mlcommons/tiny/4addd0fa08d216e20637637874e084895f289da4/LICENSE.md",
        "path": "models/vision_classification/VC02/quality_evaluation/sources/LICENSE.md",
        "expected_sha256": "0d542e0c8804e39aa7f37eb00da5a762149dc682d7829451287e11b938e94594",
        "license": "Apache-2.0"
      },
      {
        "name": "keras_v3.12.0_cifar10.py",
        "url": "https://raw.githubusercontent.com/keras-team/keras/v3.12.0/keras/src/datasets/cifar10.py",
        "path": "models/vision_classification/VC02/quality_evaluation/sources/keras_v3.12.0_cifar10.py",
        "expected_sha256": "bf651dbec372cef81ab162bda19d1a8476b513a3f4976b40240616943a45736f",
        "license": "Apache-2.0"
      },
      {
        "name": "transport_mirror_lfs_pointer.txt",
        "url": "https://huggingface.co/Peyiloo/peyiloo/raw/main/cifar-10-python.tar.gz",
        "path": "models/vision_classification/VC02/quality_evaluation/sources/transport_mirror_lfs_pointer.txt",
        "expected_sha256": "3684b5b4c4935ed2a1e722ac45675d619fe1f16930be5439803edf03047605eb",
        "license": "Provenance pointer only; dataset terms remain those of the canonical source"
      }
    ]
  },
  "models": {
    "fp32": {
      "path": "models/vision_classification/VC02/source/fp32/source_fp32.tflite",
      "source_url": "https://raw.githubusercontent.com/mlcommons/tiny/4addd0fa08d216e20637637874e084895f289da4/benchmark/training/image_classification/trained_models/pretrainedResnet.tflite",
      "expected_sha256": "b5c0046d6e0328b4956afd6baa29555a29b1f1c65bdd45aaed75b7cd484d9f79"
    },
    "public_int8": {
      "path": "models/vision_classification/VC02/source/quantized/source_public_quantized.tflite",
      "source_url": "https://raw.githubusercontent.com/mlcommons/tiny/4addd0fa08d216e20637637874e084895f289da4/benchmark/training/image_classification/trained_models/pretrainedResnet_quant.tflite",
      "expected_sha256": "3c002613d1b2475eb51dd78dfb85a546c8ae658dee71cf6ade43b022fe205415"
    }
  },
  "protocol": {
    "benchmark_id": "ic01",
    "dataset_loader": "cifar10_python_test_batch",
    "labels_path": "models/vision_classification/VC02/quality_evaluation/sources/y_labels.csv",
    "indices_path": "models/vision_classification/VC02/quality_evaluation/sources/perf_samples_idxs.npy",
    "input_shape": [1, 32, 32, 3],
    "fp32_input_mode": "raw_uint8_as_float32",
    "public_int8_input": "subtract 128 from the same raw RGB bytes, equivalently artifact scale 1 and zero point -128",
    "class_count": 10,
    "label_space": ["airplane", "automobile", "bird", "cat", "deer", "dog", "frog", "horse", "ship", "truck"],
    "label_counts": {"0": 20, "1": 20, "2": 20, "3": 20, "4": 20, "5": 20, "6": 20, "7": 20, "8": 20, "9": 20},
    "sample_order": "pinned perf_samples_idxs.npy and y_labels.csv order",
    "metric_equation": "correct argmax predictions divided by 200 official stimuli"
  },
  "metric": {
    "name": "top1_accuracy",
    "sample_count": 200,
    "acceptance_threshold": 0.85,
    "threshold_source": "models/vision_classification/VC02/quality_evaluation/sources/datasets_README.md"
  },
  "runtime": {
    "interpreter": "ai-edge-litert",
    "threads": 1,
    "batch_size": 1,
    "quantized_input_rounding": "numpy_rint_then_int8_clip",
    "output_decision": "argmax_raw_tensor"
  },
  "provenance": {
    "fetch_report": "models/vision_classification/VC02/quality_evaluation/fetch_report.json",
    "stdout_log": "models/vision_classification/VC02/quality_evaluation/logs/evaluation.stdout.log",
    "stderr_log": "models/vision_classification/VC02/quality_evaluation/logs/evaluation.stderr.log",
    "validation_report": "models/vision_classification/VC02/quality_evaluation/results/validation_report.json"
  }
}