| { | |
| "evaluation_date": "2026-10-20", | |
| "source_model": { | |
| "repository": "litert-community/embeddinggemma-2-text-vision-440m-litert-lm", | |
| "revision": "e301f74d5551b0c2641bd5cb4652a76239d5c5f8", | |
| "filename": "embeddinggemma-2-text-vision-440m.litertlm", | |
| "size": 387710976, | |
| "sha256": "92dcbea108899e5d6e30d919b0744f90d9967e80c67a4ab5503ac16d54f62eb0" | |
| }, | |
| "compact_model": { | |
| "filename": "eg2-gallery-compact.litertlm", | |
| "size": 309559296, | |
| "sha256": "a08a6bcbcba893d8e52dcf2ae05ce06847f942e3b0e5ea8b8df93f32aa23b027" | |
| }, | |
| "contract": { | |
| "tested_version": "0.18.0", | |
| "max_input_length": 256, | |
| "vision_tokens_per_image": 70, | |
| "output_size": 768, | |
| "normalize": true, | |
| "insert_special_tokens": true, | |
| "text_signatures": [ | |
| "encoder_1x128", | |
| "encoder_1x256" | |
| ], | |
| "vision_signatures": [ | |
| "vision_70", | |
| "vision_adapter_70" | |
| ], | |
| "retained_patch_coordinate_range": [ | |
| 0, | |
| 209 | |
| ] | |
| }, | |
| "inputs": { | |
| "text_queries": 73, | |
| "public_photos": 14, | |
| "synthetic_images": 6 | |
| }, | |
| "windows": { | |
| "runtime": "LiteRT-LM 0.18.0", | |
| "cpu_runs_per_model": 2, | |
| "gpu_runs_per_model": 1, | |
| "comparisons": [ | |
| { | |
| "baseline": "baseline-cpu-r1", | |
| "candidate": "compact-final-cpu-r1", | |
| "backend": "cpu", | |
| "vectors": { | |
| "queries": { | |
| "count": 73, | |
| "dimensions": 768, | |
| "all_vectors_bitwise_equal": true, | |
| "max_absolute_element_difference": 0.0 | |
| }, | |
| "images": { | |
| "count": 20, | |
| "dimensions": 768, | |
| "all_vectors_bitwise_equal": true, | |
| "max_absolute_element_difference": 0.0 | |
| } | |
| }, | |
| "text_image_cosine_max_abs_difference": 0.0, | |
| "image_image_cosine_max_abs_difference": 0.0, | |
| "all_text_image_rankings_equal": true, | |
| "all_image_image_rankings_equal": true, | |
| "threshold_decisions_equal": { | |
| "0.68": true, | |
| "0.7": true, | |
| "0.8": true, | |
| "0.85": true, | |
| "0.9": true | |
| } | |
| }, | |
| { | |
| "baseline": "baseline-cpu-r1", | |
| "candidate": "compact-final-cpu-r2", | |
| "backend": "cpu", | |
| "vectors": { | |
| "queries": { | |
| "count": 73, | |
| "dimensions": 768, | |
| "all_vectors_bitwise_equal": true, | |
| "max_absolute_element_difference": 0.0 | |
| }, | |
| "images": { | |
| "count": 20, | |
| "dimensions": 768, | |
| "all_vectors_bitwise_equal": true, | |
| "max_absolute_element_difference": 0.0 | |
| } | |
| }, | |
| "text_image_cosine_max_abs_difference": 0.0, | |
| "image_image_cosine_max_abs_difference": 0.0, | |
| "all_text_image_rankings_equal": true, | |
| "all_image_image_rankings_equal": true, | |
| "threshold_decisions_equal": { | |
| "0.68": true, | |
| "0.7": true, | |
| "0.8": true, | |
| "0.85": true, | |
| "0.9": true | |
| } | |
| }, | |
| { | |
| "baseline": "baseline-gpu-r1", | |
| "candidate": "compact-final-gpu-r1", | |
| "backend": "gpu", | |
| "vectors": { | |
| "queries": { | |
| "count": 73, | |
| "dimensions": 768, | |
| "all_vectors_bitwise_equal": true, | |
| "max_absolute_element_difference": 0.0 | |
| }, | |
| "images": { | |
| "count": 20, | |
| "dimensions": 768, | |
| "all_vectors_bitwise_equal": true, | |
| "max_absolute_element_difference": 0.0 | |
| } | |
| }, | |
| "text_image_cosine_max_abs_difference": 0.0, | |
| "image_image_cosine_max_abs_difference": 0.0, | |
| "all_text_image_rankings_equal": true, | |
| "all_image_image_rankings_equal": true, | |
| "threshold_decisions_equal": { | |
| "0.68": true, | |
| "0.7": true, | |
| "0.8": true, | |
| "0.85": true, | |
| "0.9": true | |
| } | |
| } | |
| ] | |
| }, | |
| "samsung": { | |
| "device": { | |
| "ro.product.manufacturer": "samsung", | |
| "ro.product.model": "SM-S931U", | |
| "ro.build.version.release": "16", | |
| "ro.soc.model": "SM8750" | |
| }, | |
| "runtime": "litertlm-android:0.18.0", | |
| "runs_per_model_per_backend": 1, | |
| "comparisons": { | |
| "cpu": { | |
| "all_vectors_bitwise_equal": true, | |
| "max_absolute_element_difference": 0.0, | |
| "all_text_image_rankings_equal": true, | |
| "threshold_068_equal": true | |
| }, | |
| "gpu": { | |
| "all_vectors_bitwise_equal": true, | |
| "max_absolute_element_difference": 0.0, | |
| "all_text_image_rankings_equal": true, | |
| "threshold_068_equal": true | |
| } | |
| }, | |
| "comparison_scope": "Original vs compact on the same backend; not CPU-vs-GPU bitwise equality." | |
| }, | |
| "limitations": [ | |
| "Regression cohort, not an independent general accuracy benchmark.", | |
| "Native model loading and embedding only; no full photo-library app-flow validation.", | |
| "No iOS runtime validation for the compact bundle.", | |
| "Only the published 256-input-token/70-vision-token/768-dimension contract is validated." | |
| ] | |
| } | |