{ "base_model": "Reza2kn/Bina-0.4-Rizeh", "base_revision": "51b8c314b0552b1884c1f0e28016cb12a143f637", "scope": "Five CTC readers: frozen300 preprocessed crops each; detectors: procedural-dev heatmaps only", "reader_targets": [ { "role": "short_labels", "runtime": "browser-wasm", "source_checkpoint_sha256": "367084525dc644ef6d69ad981819af07e3c6a41421359594a54ccc989c2fb90a", "model_sha256": "cdb2d08050b2a7cbaff7154d39b6bdfcf0fdcc601f9180641864cf1ab193c4cf", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9873118914133963, "crops_per_second": 0.9867245422170867, "buckets": { "english": { "cases": 50, "character_parity": 0.9986076301865776, "exact_line_parity": 0.9, "candidate_CER": 0.001392757660167131, "source_CER": 0.0011142061281337048, "CER_delta": 0.0002785515320334262 }, "handwriting": { "cases": 50, "character_parity": 0.995895330938943, "exact_line_parity": 0.86, "candidate_CER": 0.011253196930946292, "source_CER": 0.010741687979539642, "CER_delta": 0.0005115089514066496 }, "mixed": { "cases": 50, "character_parity": 0.9961089494163424, "exact_line_parity": 0.94, "candidate_CER": 0.010104011887072809, "source_CER": 0.00861812778603269, "CER_delta": 0.0014858841010401188 }, "newspaper": { "cases": 50, "character_parity": 0.9995359628770302, "exact_line_parity": 0.98, "candidate_CER": 0.027829313543599257, "source_CER": 0.02736549165120594, "CER_delta": 0.00046382189239332097 }, "printed": { "cases": 50, "character_parity": 0.9963476990504018, "exact_line_parity": 0.9, "candidate_CER": 0.0058309037900874635, "source_CER": 0.00510204081632653, "CER_delta": 0.0007288629737609329 }, "synthetic": { "cases": 50, "character_parity": 0.9873118914133963, "exact_line_parity": 0.8, "candidate_CER": 0.05410821643286573, "source_CER": 0.05468079015173204, "CER_delta": -0.000572573718866304 } }, "execution": { "threads": 2, "timing_scope": "Warm bucket, preprocessed crop to compact outputs, exact result cache disabled; concurrent Mac validation jobs may contend" } }, { "role": "short_labels", "runtime": "onnx-linux-cpu-hqq4-block32", "source_checkpoint_sha256": "367084525dc644ef6d69ad981819af07e3c6a41421359594a54ccc989c2fb90a", "model_sha256": "e0954ac2ffc4b574b9c6a2c5a1d09117b5238b9ec8a4e97b83b5f46f963fc817", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9873118914133963, "crops_per_second": 14.501852935194389, "buckets": { "english": { "cases": 50, "character_parity": 0.999443052074631, "exact_line_parity": 0.96, "candidate_CER": 0.0011142061281337048, "source_CER": 0.0011142061281337048, "CER_delta": 0.0 }, "handwriting": { "cases": 50, "character_parity": 0.9943560800410467, "exact_line_parity": 0.88, "candidate_CER": 0.010741687979539642, "source_CER": 0.010741687979539642, "CER_delta": 0.0 }, "mixed": { "cases": 50, "character_parity": 0.9940137683328345, "exact_line_parity": 0.92, "candidate_CER": 0.012481426448737, "source_CER": 0.00861812778603269, "CER_delta": 0.003863298662704309 }, "newspaper": { "cases": 50, "character_parity": 0.9981438515081207, "exact_line_parity": 0.94, "candidate_CER": 0.02736549165120594, "source_CER": 0.02736549165120594, "CER_delta": 0.0 }, "printed": { "cases": 50, "character_parity": 0.9992695398100804, "exact_line_parity": 0.98, "candidate_CER": 0.0058309037900874635, "source_CER": 0.00510204081632653, "CER_delta": 0.0007288629737609329 }, "synthetic": { "cases": 50, "character_parity": 0.9873118914133963, "exact_line_parity": 0.78, "candidate_CER": 0.057543658746063556, "source_CER": 0.05468079015173204, "CER_delta": 0.0028628685943315204 } }, "execution": { "runtime_version": "1.31.0", "platform": "Linux-7.0.0-31-generic-x86_64-with-glibc2.43", "threads": 2, "batch": 1, "timing_scope": "Preprocessed crop -> compact tokens/confidence; excludes model load, preprocessing, CTC/bidi decode; caches disabled; first shape warmed only", "provider": "CPUExecutionProvider", "actual_profile_providers": { "CPUExecutionProvider": 371 }, "baseline_sha256": "8228c8ed18fd274bf93bc849652bb20929ea7813f878b24e3f2262e46f8c288d", "manifest_sha256": "7e7049a3b56323588ae1f6bd38b1753afc2b1149d275c6f0f07c6993be991978" } }, { "role": "short_labels", "runtime": "coreml-w4-block16", "source_checkpoint_sha256": "367084525dc644ef6d69ad981819af07e3c6a41421359594a54ccc989c2fb90a", "model_sha256": null, "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9846562407789908, "crops_per_second": 37.9281905430233, "buckets": { "english": { "cases": 50, "character_parity": 0.9991645781119465, "exact_line_parity": 0.94, "candidate_CER": 0.001392757660167131, "source_CER": 0.0011142061281337048, "CER_delta": 0.0002785515320334262 }, "handwriting": { "cases": 50, "character_parity": 0.9938429964084146, "exact_line_parity": 0.78, "candidate_CER": 0.012276214833759591, "source_CER": 0.010741687979539642, "CER_delta": 0.001534526854219949 }, "mixed": { "cases": 50, "character_parity": 0.9961089494163424, "exact_line_parity": 0.88, "candidate_CER": 0.00950965824665676, "source_CER": 0.00861812778603269, "CER_delta": 0.0008915304606240713 }, "newspaper": { "cases": 50, "character_parity": 0.9986078886310905, "exact_line_parity": 0.94, "candidate_CER": 0.026901669758812616, "source_CER": 0.02736549165120594, "CER_delta": -0.00046382189239332097 }, "printed": { "cases": 50, "character_parity": 0.9963476990504018, "exact_line_parity": 0.92, "candidate_CER": 0.006559766763848397, "source_CER": 0.00510204081632653, "CER_delta": 0.0014577259475218659 }, "synthetic": { "cases": 50, "character_parity": 0.9846562407789908, "exact_line_parity": 0.76, "candidate_CER": 0.05410821643286573, "source_CER": 0.05468079015173204, "CER_delta": -0.000572573718866304 } }, "execution": { "platform": "macOS-26.5.1-arm64-arm-64bit", "threads": 2, "batch": 1, "timing_scope": "Preprocessed crop -> compact tokens/confidence; excludes model load, preprocessing, CTC/bidi decode; caches disabled; first shape warmed only", "provider": "CPUExecutionProvider", "baseline_sha256": "8228c8ed18fd274bf93bc849652bb20929ea7813f878b24e3f2262e46f8c288d", "manifest_sha256": "7e7049a3b56323588ae1f6bd38b1753afc2b1149d275c6f0f07c6993be991978" } }, { "role": "short_labels", "runtime": "onnx-hqq4-block32", "source_checkpoint_sha256": "367084525dc644ef6d69ad981819af07e3c6a41421359594a54ccc989c2fb90a", "model_sha256": "e0954ac2ffc4b574b9c6a2c5a1d09117b5238b9ec8a4e97b83b5f46f963fc817", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9873118914133963, "crops_per_second": 7.6651239617737925, "buckets": { "english": { "cases": 50, "character_parity": 0.999443052074631, "exact_line_parity": 0.96, "candidate_CER": 0.0011142061281337048, "source_CER": 0.0011142061281337048, "CER_delta": 0.0 }, "handwriting": { "cases": 50, "character_parity": 0.9943560800410467, "exact_line_parity": 0.88, "candidate_CER": 0.010741687979539642, "source_CER": 0.010741687979539642, "CER_delta": 0.0 }, "mixed": { "cases": 50, "character_parity": 0.9940137683328345, "exact_line_parity": 0.92, "candidate_CER": 0.012481426448737, "source_CER": 0.00861812778603269, "CER_delta": 0.003863298662704309 }, "newspaper": { "cases": 50, "character_parity": 0.9981438515081207, "exact_line_parity": 0.94, "candidate_CER": 0.02736549165120594, "source_CER": 0.02736549165120594, "CER_delta": 0.0 }, "printed": { "cases": 50, "character_parity": 0.9992695398100804, "exact_line_parity": 0.98, "candidate_CER": 0.0058309037900874635, "source_CER": 0.00510204081632653, "CER_delta": 0.0007288629737609329 }, "synthetic": { "cases": 50, "character_parity": 0.9873118914133963, "exact_line_parity": 0.78, "candidate_CER": 0.057543658746063556, "source_CER": 0.05468079015173204, "CER_delta": 0.0028628685943315204 } }, "execution": { "runtime_version": "1.31.0", "platform": "macOS-26.5.1-arm64-arm-64bit", "threads": 2, "batch": 1, "timing_scope": "Preprocessed crop -> compact tokens/confidence; excludes model load, preprocessing, CTC/bidi decode; caches disabled; first shape warmed only", "provider": "CPUExecutionProvider", "actual_profile_providers": { "CPUExecutionProvider": 370 }, "baseline_sha256": "8228c8ed18fd274bf93bc849652bb20929ea7813f878b24e3f2262e46f8c288d", "manifest_sha256": "7e7049a3b56323588ae1f6bd38b1753afc2b1149d275c6f0f07c6993be991978" } }, { "role": "short_labels", "runtime": "tract-linux-cuda-w4-block16", "source_checkpoint_sha256": "367084525dc644ef6d69ad981819af07e3c6a41421359594a54ccc989c2fb90a", "model_sha256": "cdb2d08050b2a7cbaff7154d39b6bdfcf0fdcc601f9180641864cf1ab193c4cf", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9873118914133963, "crops_per_second": 242.5290926015559, "buckets": { "english": { "cases": 50, "character_parity": 0.9986076301865776, "exact_line_parity": 0.9, "candidate_CER": 0.001392757660167131, "source_CER": 0.0011142061281337048, "CER_delta": 0.0002785515320334262 }, "handwriting": { "cases": 50, "character_parity": 0.995895330938943, "exact_line_parity": 0.86, "candidate_CER": 0.011253196930946292, "source_CER": 0.010741687979539642, "CER_delta": 0.0005115089514066496 }, "mixed": { "cases": 50, "character_parity": 0.9961089494163424, "exact_line_parity": 0.94, "candidate_CER": 0.010104011887072809, "source_CER": 0.00861812778603269, "CER_delta": 0.0014858841010401188 }, "newspaper": { "cases": 50, "character_parity": 0.9995359628770302, "exact_line_parity": 0.98, "candidate_CER": 0.027829313543599257, "source_CER": 0.02736549165120594, "CER_delta": 0.00046382189239332097 }, "printed": { "cases": 50, "character_parity": 0.9963476990504018, "exact_line_parity": 0.9, "candidate_CER": 0.0058309037900874635, "source_CER": 0.00510204081632653, "CER_delta": 0.0007288629737609329 }, "synthetic": { "cases": 50, "character_parity": 0.9873118914133963, "exact_line_parity": 0.8, "candidate_CER": 0.05410821643286573, "source_CER": 0.05468079015173204, "CER_delta": -0.000572573718866304 } }, "execution": { "timing_scope": "Batch1, native model time after plan warmup; excludes plan preparation, input IO, CTC/bidi; exact-result cache off", "version": "0.23.9-pre", "plan_cache_hits": 296 } }, { "role": "short_labels", "runtime": "browser-webgpu", "source_checkpoint_sha256": "367084525dc644ef6d69ad981819af07e3c6a41421359594a54ccc989c2fb90a", "model_sha256": "cdb2d08050b2a7cbaff7154d39b6bdfcf0fdcc601f9180641864cf1ab193c4cf", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9873118914133963, "crops_per_second": 8.822682174510609, "buckets": { "english": { "cases": 50, "character_parity": 0.9986076301865776, "exact_line_parity": 0.9, "candidate_CER": 0.001392757660167131, "source_CER": 0.0011142061281337048, "CER_delta": 0.0002785515320334262 }, "handwriting": { "cases": 50, "character_parity": 0.995895330938943, "exact_line_parity": 0.86, "candidate_CER": 0.011253196930946292, "source_CER": 0.010741687979539642, "CER_delta": 0.0005115089514066496 }, "mixed": { "cases": 50, "character_parity": 0.9961089494163424, "exact_line_parity": 0.94, "candidate_CER": 0.010104011887072809, "source_CER": 0.00861812778603269, "CER_delta": 0.0014858841010401188 }, "newspaper": { "cases": 50, "character_parity": 0.9995359628770302, "exact_line_parity": 0.98, "candidate_CER": 0.027829313543599257, "source_CER": 0.02736549165120594, "CER_delta": 0.00046382189239332097 }, "printed": { "cases": 50, "character_parity": 0.9963476990504018, "exact_line_parity": 0.9, "candidate_CER": 0.0058309037900874635, "source_CER": 0.00510204081632653, "CER_delta": 0.0007288629737609329 }, "synthetic": { "cases": 50, "character_parity": 0.9873118914133963, "exact_line_parity": 0.8, "candidate_CER": 0.05410821643286573, "source_CER": 0.05468079015173204, "CER_delta": -0.000572573718866304 } }, "execution": { "threads": 2, "timing_scope": "Warm bucket, preprocessed crop to compact outputs, exact result cache disabled; concurrent Mac validation jobs may contend" } }, { "role": "short_labels", "runtime": "onnx-linux-cuda-w4-block16", "source_checkpoint_sha256": "367084525dc644ef6d69ad981819af07e3c6a41421359594a54ccc989c2fb90a", "model_sha256": "cdb2d08050b2a7cbaff7154d39b6bdfcf0fdcc601f9180641864cf1ab193c4cf", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9873118914133963, "crops_per_second": 342.76497929340195, "buckets": { "english": { "cases": 50, "character_parity": 0.9986076301865776, "exact_line_parity": 0.9, "candidate_CER": 0.001392757660167131, "source_CER": 0.0011142061281337048, "CER_delta": 0.0002785515320334262 }, "handwriting": { "cases": 50, "character_parity": 0.995895330938943, "exact_line_parity": 0.86, "candidate_CER": 0.011253196930946292, "source_CER": 0.010741687979539642, "CER_delta": 0.0005115089514066496 }, "mixed": { "cases": 50, "character_parity": 0.9961089494163424, "exact_line_parity": 0.94, "candidate_CER": 0.010104011887072809, "source_CER": 0.00861812778603269, "CER_delta": 0.0014858841010401188 }, "newspaper": { "cases": 50, "character_parity": 0.9995359628770302, "exact_line_parity": 0.98, "candidate_CER": 0.027829313543599257, "source_CER": 0.02736549165120594, "CER_delta": 0.00046382189239332097 }, "printed": { "cases": 50, "character_parity": 0.9963476990504018, "exact_line_parity": 0.9, "candidate_CER": 0.0058309037900874635, "source_CER": 0.00510204081632653, "CER_delta": 0.0007288629737609329 }, "synthetic": { "cases": 50, "character_parity": 0.9873118914133963, "exact_line_parity": 0.8, "candidate_CER": 0.05410821643286573, "source_CER": 0.05468079015173204, "CER_delta": -0.000572573718866304 } }, "execution": { "runtime_version": "1.23.2", "platform": "Linux-7.0.0-31-generic-x86_64-with-glibc2.43", "threads": 2, "batch": 1, "timing_scope": "Preprocessed crop -> compact tokens/confidence; excludes model load, preprocessing, CTC/bidi decode; caches disabled; first shape warmed only", "provider": "CUDAExecutionProvider", "actual_profile_providers": { "CUDAExecutionProvider": 448, "CPUExecutionProvider": 3 }, "baseline_sha256": "8228c8ed18fd274bf93bc849652bb20929ea7813f878b24e3f2262e46f8c288d", "manifest_sha256": "7e7049a3b56323588ae1f6bd38b1753afc2b1149d275c6f0f07c6993be991978" } }, { "role": "short_labels", "runtime": "tract-cpu-w4-block16", "source_checkpoint_sha256": "367084525dc644ef6d69ad981819af07e3c6a41421359594a54ccc989c2fb90a", "model_sha256": "cdb2d08050b2a7cbaff7154d39b6bdfcf0fdcc601f9180641864cf1ab193c4cf", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9873118914133963, "crops_per_second": 21.854343609292783, "buckets": { "english": { "cases": 50, "character_parity": 0.9986076301865776, "exact_line_parity": 0.9, "candidate_CER": 0.001392757660167131, "source_CER": 0.0011142061281337048, "CER_delta": 0.0002785515320334262 }, "handwriting": { "cases": 50, "character_parity": 0.995895330938943, "exact_line_parity": 0.86, "candidate_CER": 0.011253196930946292, "source_CER": 0.010741687979539642, "CER_delta": 0.0005115089514066496 }, "mixed": { "cases": 50, "character_parity": 0.9961089494163424, "exact_line_parity": 0.94, "candidate_CER": 0.010104011887072809, "source_CER": 0.00861812778603269, "CER_delta": 0.0014858841010401188 }, "newspaper": { "cases": 50, "character_parity": 0.9995359628770302, "exact_line_parity": 0.98, "candidate_CER": 0.027829313543599257, "source_CER": 0.02736549165120594, "CER_delta": 0.00046382189239332097 }, "printed": { "cases": 50, "character_parity": 0.9963476990504018, "exact_line_parity": 0.9, "candidate_CER": 0.0058309037900874635, "source_CER": 0.00510204081632653, "CER_delta": 0.0007288629737609329 }, "synthetic": { "cases": 50, "character_parity": 0.9873118914133963, "exact_line_parity": 0.8, "candidate_CER": 0.05410821643286573, "source_CER": 0.05468079015173204, "CER_delta": -0.000572573718866304 } }, "execution": { "timing_scope": "Batch1, native model time after plan warmup; excludes plan preparation, input IO, CTC/bidi; exact-result cache off", "version": "0.23.9-pre", "plan_cache_hits": 296 } }, { "role": "short_labels", "runtime": "onnx-w4-block16", "source_checkpoint_sha256": "367084525dc644ef6d69ad981819af07e3c6a41421359594a54ccc989c2fb90a", "model_sha256": "cdb2d08050b2a7cbaff7154d39b6bdfcf0fdcc601f9180641864cf1ab193c4cf", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9861316022425495, "crops_per_second": 8.339678565514452, "buckets": { "english": { "cases": 50, "character_parity": 0.9986076301865776, "exact_line_parity": 0.9, "candidate_CER": 0.001392757660167131, "source_CER": 0.0011142061281337048, "CER_delta": 0.0002785515320334262 }, "handwriting": { "cases": 50, "character_parity": 0.995895330938943, "exact_line_parity": 0.86, "candidate_CER": 0.011253196930946292, "source_CER": 0.010741687979539642, "CER_delta": 0.0005115089514066496 }, "mixed": { "cases": 50, "character_parity": 0.9961089494163424, "exact_line_parity": 0.94, "candidate_CER": 0.010104011887072809, "source_CER": 0.00861812778603269, "CER_delta": 0.0014858841010401188 }, "newspaper": { "cases": 50, "character_parity": 0.9995359628770302, "exact_line_parity": 0.98, "candidate_CER": 0.027829313543599257, "source_CER": 0.02736549165120594, "CER_delta": 0.00046382189239332097 }, "printed": { "cases": 50, "character_parity": 0.9963476990504018, "exact_line_parity": 0.9, "candidate_CER": 0.0058309037900874635, "source_CER": 0.00510204081632653, "CER_delta": 0.0007288629737609329 }, "synthetic": { "cases": 50, "character_parity": 0.9861316022425495, "exact_line_parity": 0.8, "candidate_CER": 0.05353564271399943, "source_CER": 0.05468079015173204, "CER_delta": -0.001145147437732608 } }, "execution": { "runtime_version": "1.31.0", "platform": "macOS-26.5.1-arm64-arm-64bit", "threads": 2, "batch": 1, "timing_scope": "Preprocessed crop -> compact tokens/confidence; excludes model load, preprocessing, CTC/bidi decode; caches disabled; first shape warmed only", "provider": "CPUExecutionProvider", "actual_profile_providers": { "CPUExecutionProvider": 370 }, "baseline_sha256": "8228c8ed18fd274bf93bc849652bb20929ea7813f878b24e3f2262e46f8c288d", "manifest_sha256": "7e7049a3b56323588ae1f6bd38b1753afc2b1149d275c6f0f07c6993be991978" } }, { "role": "short_labels", "runtime": "litert-dynamic4-block32", "source_checkpoint_sha256": "367084525dc644ef6d69ad981819af07e3c6a41421359594a54ccc989c2fb90a", "model_sha256": "3e946ddfd8108019e56bee58462073b9f8daa038ad5cc6bda569bef95477639f", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9834759516081439, "crops_per_second": 45.07273258563781, "buckets": { "english": { "cases": 50, "character_parity": 0.9980506822612085, "exact_line_parity": 0.9, "candidate_CER": 0.0019498607242339832, "source_CER": 0.0011142061281337048, "CER_delta": 0.0008356545961002785 }, "handwriting": { "cases": 50, "character_parity": 0.9892252437147255, "exact_line_parity": 0.72, "candidate_CER": 0.01381074168797954, "source_CER": 0.010741687979539642, "CER_delta": 0.003069053708439898 }, "mixed": { "cases": 50, "character_parity": 0.9964082609997007, "exact_line_parity": 0.92, "candidate_CER": 0.009806835066864784, "source_CER": 0.00861812778603269, "CER_delta": 0.001188707280832095 }, "newspaper": { "cases": 50, "character_parity": 0.9967517401392111, "exact_line_parity": 0.9, "candidate_CER": 0.029684601113172542, "source_CER": 0.02736549165120594, "CER_delta": 0.002319109461966605 }, "printed": { "cases": 50, "character_parity": 0.9934258582907232, "exact_line_parity": 0.84, "candidate_CER": 0.008746355685131196, "source_CER": 0.00510204081632653, "CER_delta": 0.0036443148688046646 }, "synthetic": { "cases": 50, "character_parity": 0.9834759516081439, "exact_line_parity": 0.64, "candidate_CER": 0.057543658746063556, "source_CER": 0.05468079015173204, "CER_delta": 0.0028628685943315204 } }, "execution": { "platform": "Linux-7.0.0-31-generic-x86_64-with-glibc2.43", "threads": 2, "batch": 1, "timing_scope": "Preprocessed crop -> compact tokens/confidence; excludes model load, preprocessing, CTC/bidi decode; caches disabled; first shape warmed only", "provider": "CPUExecutionProvider", "baseline_sha256": "8228c8ed18fd274bf93bc849652bb20929ea7813f878b24e3f2262e46f8c288d", "manifest_sha256": "7e7049a3b56323588ae1f6bd38b1753afc2b1149d275c6f0f07c6993be991978" } }, { "role": "short_labels", "runtime": "tract-metal-w4-block16", "source_checkpoint_sha256": "367084525dc644ef6d69ad981819af07e3c6a41421359594a54ccc989c2fb90a", "model_sha256": "cdb2d08050b2a7cbaff7154d39b6bdfcf0fdcc601f9180641864cf1ab193c4cf", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9873118914133963, "crops_per_second": 27.759534227305558, "buckets": { "english": { "cases": 50, "character_parity": 0.9986076301865776, "exact_line_parity": 0.9, "candidate_CER": 0.001392757660167131, "source_CER": 0.0011142061281337048, "CER_delta": 0.0002785515320334262 }, "handwriting": { "cases": 50, "character_parity": 0.995895330938943, "exact_line_parity": 0.86, "candidate_CER": 0.011253196930946292, "source_CER": 0.010741687979539642, "CER_delta": 0.0005115089514066496 }, "mixed": { "cases": 50, "character_parity": 0.9961089494163424, "exact_line_parity": 0.94, "candidate_CER": 0.010104011887072809, "source_CER": 0.00861812778603269, "CER_delta": 0.0014858841010401188 }, "newspaper": { "cases": 50, "character_parity": 0.9995359628770302, "exact_line_parity": 0.98, "candidate_CER": 0.027829313543599257, "source_CER": 0.02736549165120594, "CER_delta": 0.00046382189239332097 }, "printed": { "cases": 50, "character_parity": 0.9963476990504018, "exact_line_parity": 0.9, "candidate_CER": 0.0058309037900874635, "source_CER": 0.00510204081632653, "CER_delta": 0.0007288629737609329 }, "synthetic": { "cases": 50, "character_parity": 0.9873118914133963, "exact_line_parity": 0.8, "candidate_CER": 0.05410821643286573, "source_CER": 0.05468079015173204, "CER_delta": -0.000572573718866304 } }, "execution": { "timing_scope": "Batch1, native model time after plan warmup; excludes plan preparation, input IO, CTC/bidi; exact-result cache off", "version": "0.23.9-pre", "plan_cache_hits": 296 } }, { "role": "body", "runtime": "browser-wasm", "source_checkpoint_sha256": "036be44e87c05759c18297426d2d531b0c3ac04af172886071de2ecdd6703981", "model_sha256": "2cb6c60a23a2e9948b9df9679de0f8b2c849dd81bc54abbf4b23f64148edf574", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9870473947600824, "crops_per_second": 0.909649944111392, "buckets": { "english": { "cases": 50, "character_parity": 0.9991645781119465, "exact_line_parity": 0.94, "candidate_CER": 0.0008356545961002785, "source_CER": 0.0011142061281337048, "CER_delta": -0.0002785515320334262 }, "handwriting": { "cases": 50, "character_parity": 0.9948638931689779, "exact_line_parity": 0.84, "candidate_CER": 0.010230179028132993, "source_CER": 0.010741687979539642, "CER_delta": -0.0005115089514066496 }, "mixed": { "cases": 50, "character_parity": 0.9952095808383233, "exact_line_parity": 0.92, "candidate_CER": 0.010104011887072809, "source_CER": 0.00861812778603269, "CER_delta": 0.0014858841010401188 }, "newspaper": { "cases": 50, "character_parity": 0.9995361781076066, "exact_line_parity": 0.98, "candidate_CER": 0.026901669758812616, "source_CER": 0.026437847866419294, "CER_delta": 0.00046382189239332097 }, "printed": { "cases": 50, "character_parity": 0.9963476990504018, "exact_line_parity": 0.9, "candidate_CER": 0.006559766763848397, "source_CER": 0.00510204081632653, "CER_delta": 0.0014577259475218659 }, "synthetic": { "cases": 50, "character_parity": 0.9870473947600824, "exact_line_parity": 0.78, "candidate_CER": 0.05382192957343258, "source_CER": 0.05410821643286573, "CER_delta": -0.000286286859433152 } }, "execution": { "threads": 2, "timing_scope": "Warm bucket, preprocessed crop to compact outputs, exact result cache disabled; concurrent Mac validation jobs may contend" } }, { "role": "body", "runtime": "onnx-linux-cpu-hqq4-block32", "source_checkpoint_sha256": "036be44e87c05759c18297426d2d531b0c3ac04af172886071de2ecdd6703981", "model_sha256": "c8b9677ce060c5e50edfcc7e78364b0219c78b4bbb84514a77ca71757fcd3f5f", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9885192817191639, "crops_per_second": 14.416615669727436, "buckets": { "english": { "cases": 50, "character_parity": 0.999443052074631, "exact_line_parity": 0.96, "candidate_CER": 0.0011142061281337048, "source_CER": 0.0011142061281337048, "CER_delta": 0.0 }, "handwriting": { "cases": 50, "character_parity": 0.9917822290703646, "exact_line_parity": 0.82, "candidate_CER": 0.012276214833759591, "source_CER": 0.010741687979539642, "CER_delta": 0.001534526854219949 }, "mixed": { "cases": 50, "character_parity": 0.9940119760479041, "exact_line_parity": 0.92, "candidate_CER": 0.01337295690936107, "source_CER": 0.00861812778603269, "CER_delta": 0.00475482912332838 }, "newspaper": { "cases": 50, "character_parity": 0.9995361781076066, "exact_line_parity": 0.98, "candidate_CER": 0.026901669758812616, "source_CER": 0.026437847866419294, "CER_delta": 0.00046382189239332097 }, "printed": { "cases": 50, "character_parity": 0.9985390796201608, "exact_line_parity": 0.96, "candidate_CER": 0.006559766763848397, "source_CER": 0.00510204081632653, "CER_delta": 0.0014577259475218659 }, "synthetic": { "cases": 50, "character_parity": 0.9885192817191639, "exact_line_parity": 0.8, "candidate_CER": 0.05639851130833095, "source_CER": 0.05410821643286573, "CER_delta": 0.002290294875465216 } }, "execution": { "runtime_version": "1.31.0", "platform": "Linux-7.0.0-31-generic-x86_64-with-glibc2.43", "threads": 2, "batch": 1, "timing_scope": "Preprocessed crop -> compact tokens/confidence; excludes model load, preprocessing, CTC/bidi decode; caches disabled; first shape warmed only", "provider": "CPUExecutionProvider", "actual_profile_providers": { "CPUExecutionProvider": 371 }, "baseline_sha256": "c9228d9d65efae590dc9c307f4909cea4d436929cc6887875ff5d2cbd3f7879b", "manifest_sha256": "7e7049a3b56323588ae1f6bd38b1753afc2b1149d275c6f0f07c6993be991978" } }, { "role": "body", "runtime": "coreml-w4-block16", "source_checkpoint_sha256": "036be44e87c05759c18297426d2d531b0c3ac04af172886071de2ecdd6703981", "model_sha256": null, "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9873417721518988, "crops_per_second": 51.39025184516772, "buckets": { "english": { "cases": 50, "character_parity": 0.9991645781119465, "exact_line_parity": 0.94, "candidate_CER": 0.001392757660167131, "source_CER": 0.0011142061281337048, "CER_delta": 0.0002785515320334262 }, "handwriting": { "cases": 50, "character_parity": 0.9943502824858758, "exact_line_parity": 0.82, "candidate_CER": 0.01432225063938619, "source_CER": 0.010741687979539642, "CER_delta": 0.0035805626598465474 }, "mixed": { "cases": 50, "character_parity": 0.9949101796407186, "exact_line_parity": 0.86, "candidate_CER": 0.00950965824665676, "source_CER": 0.00861812778603269, "CER_delta": 0.0008915304606240713 }, "newspaper": { "cases": 50, "character_parity": 0.9990723562152134, "exact_line_parity": 0.96, "candidate_CER": 0.02736549165120594, "source_CER": 0.026437847866419294, "CER_delta": 0.0009276437847866419 }, "printed": { "cases": 50, "character_parity": 0.9963476990504018, "exact_line_parity": 0.92, "candidate_CER": 0.006559766763848397, "source_CER": 0.00510204081632653, "CER_delta": 0.0014577259475218659 }, "synthetic": { "cases": 50, "character_parity": 0.9873417721518988, "exact_line_parity": 0.78, "candidate_CER": 0.0566847981677641, "source_CER": 0.05410821643286573, "CER_delta": 0.002576581734898368 } }, "execution": { "platform": "macOS-26.5.1-arm64-arm-64bit", "threads": 2, "batch": 1, "timing_scope": "Preprocessed crop -> compact tokens/confidence; excludes model load, preprocessing, CTC/bidi decode; caches disabled; first shape warmed only", "provider": "CPUExecutionProvider", "baseline_sha256": "c9228d9d65efae590dc9c307f4909cea4d436929cc6887875ff5d2cbd3f7879b", "manifest_sha256": "7e7049a3b56323588ae1f6bd38b1753afc2b1149d275c6f0f07c6993be991978" } }, { "role": "body", "runtime": "onnx-hqq4-block32", "source_checkpoint_sha256": "036be44e87c05759c18297426d2d531b0c3ac04af172886071de2ecdd6703981", "model_sha256": "c8b9677ce060c5e50edfcc7e78364b0219c78b4bbb84514a77ca71757fcd3f5f", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9885192817191639, "crops_per_second": 7.691890388291468, "buckets": { "english": { "cases": 50, "character_parity": 0.999443052074631, "exact_line_parity": 0.96, "candidate_CER": 0.0011142061281337048, "source_CER": 0.0011142061281337048, "CER_delta": 0.0 }, "handwriting": { "cases": 50, "character_parity": 0.9917822290703646, "exact_line_parity": 0.82, "candidate_CER": 0.012276214833759591, "source_CER": 0.010741687979539642, "CER_delta": 0.001534526854219949 }, "mixed": { "cases": 50, "character_parity": 0.9940119760479041, "exact_line_parity": 0.92, "candidate_CER": 0.01337295690936107, "source_CER": 0.00861812778603269, "CER_delta": 0.00475482912332838 }, "newspaper": { "cases": 50, "character_parity": 0.9995361781076066, "exact_line_parity": 0.98, "candidate_CER": 0.026901669758812616, "source_CER": 0.026437847866419294, "CER_delta": 0.00046382189239332097 }, "printed": { "cases": 50, "character_parity": 0.9985390796201608, "exact_line_parity": 0.96, "candidate_CER": 0.006559766763848397, "source_CER": 0.00510204081632653, "CER_delta": 0.0014577259475218659 }, "synthetic": { "cases": 50, "character_parity": 0.9885192817191639, "exact_line_parity": 0.8, "candidate_CER": 0.05639851130833095, "source_CER": 0.05410821643286573, "CER_delta": 0.002290294875465216 } }, "execution": { "runtime_version": "1.31.0", "platform": "macOS-26.5.1-arm64-arm-64bit", "threads": 2, "batch": 1, "timing_scope": "Preprocessed crop -> compact tokens/confidence; excludes model load, preprocessing, CTC/bidi decode; caches disabled; first shape warmed only", "provider": "CPUExecutionProvider", "actual_profile_providers": { "CPUExecutionProvider": 370 }, "baseline_sha256": "c9228d9d65efae590dc9c307f4909cea4d436929cc6887875ff5d2cbd3f7879b", "manifest_sha256": "7e7049a3b56323588ae1f6bd38b1753afc2b1149d275c6f0f07c6993be991978" } }, { "role": "body", "runtime": "tract-linux-cuda-w4-block16", "source_checkpoint_sha256": "036be44e87c05759c18297426d2d531b0c3ac04af172886071de2ecdd6703981", "model_sha256": "2cb6c60a23a2e9948b9df9679de0f8b2c849dd81bc54abbf4b23f64148edf574", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9870473947600824, "crops_per_second": 230.44644301836308, "buckets": { "english": { "cases": 50, "character_parity": 0.9991645781119465, "exact_line_parity": 0.94, "candidate_CER": 0.0008356545961002785, "source_CER": 0.0011142061281337048, "CER_delta": -0.0002785515320334262 }, "handwriting": { "cases": 50, "character_parity": 0.9948638931689779, "exact_line_parity": 0.84, "candidate_CER": 0.010230179028132993, "source_CER": 0.010741687979539642, "CER_delta": -0.0005115089514066496 }, "mixed": { "cases": 50, "character_parity": 0.9952095808383233, "exact_line_parity": 0.92, "candidate_CER": 0.010104011887072809, "source_CER": 0.00861812778603269, "CER_delta": 0.0014858841010401188 }, "newspaper": { "cases": 50, "character_parity": 0.9995361781076066, "exact_line_parity": 0.98, "candidate_CER": 0.026901669758812616, "source_CER": 0.026437847866419294, "CER_delta": 0.00046382189239332097 }, "printed": { "cases": 50, "character_parity": 0.9963476990504018, "exact_line_parity": 0.9, "candidate_CER": 0.006559766763848397, "source_CER": 0.00510204081632653, "CER_delta": 0.0014577259475218659 }, "synthetic": { "cases": 50, "character_parity": 0.9870473947600824, "exact_line_parity": 0.78, "candidate_CER": 0.05382192957343258, "source_CER": 0.05410821643286573, "CER_delta": -0.000286286859433152 } }, "execution": { "timing_scope": "Batch1, native model time after plan warmup; excludes plan preparation, input IO, CTC/bidi; exact-result cache off", "version": "0.23.9-pre", "plan_cache_hits": 296 } }, { "role": "body", "runtime": "browser-webgpu", "source_checkpoint_sha256": "036be44e87c05759c18297426d2d531b0c3ac04af172886071de2ecdd6703981", "model_sha256": "2cb6c60a23a2e9948b9df9679de0f8b2c849dd81bc54abbf4b23f64148edf574", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9870473947600824, "crops_per_second": 8.434610749622383, "buckets": { "english": { "cases": 50, "character_parity": 0.9991645781119465, "exact_line_parity": 0.94, "candidate_CER": 0.0008356545961002785, "source_CER": 0.0011142061281337048, "CER_delta": -0.0002785515320334262 }, "handwriting": { "cases": 50, "character_parity": 0.9948638931689779, "exact_line_parity": 0.84, "candidate_CER": 0.010230179028132993, "source_CER": 0.010741687979539642, "CER_delta": -0.0005115089514066496 }, "mixed": { "cases": 50, "character_parity": 0.9952095808383233, "exact_line_parity": 0.92, "candidate_CER": 0.010104011887072809, "source_CER": 0.00861812778603269, "CER_delta": 0.0014858841010401188 }, "newspaper": { "cases": 50, "character_parity": 0.9995361781076066, "exact_line_parity": 0.98, "candidate_CER": 0.026901669758812616, "source_CER": 0.026437847866419294, "CER_delta": 0.00046382189239332097 }, "printed": { "cases": 50, "character_parity": 0.9963476990504018, "exact_line_parity": 0.9, "candidate_CER": 0.006559766763848397, "source_CER": 0.00510204081632653, "CER_delta": 0.0014577259475218659 }, "synthetic": { "cases": 50, "character_parity": 0.9870473947600824, "exact_line_parity": 0.78, "candidate_CER": 0.05382192957343258, "source_CER": 0.05410821643286573, "CER_delta": -0.000286286859433152 } }, "execution": { "threads": 2, "timing_scope": "Warm bucket, preprocessed crop to compact outputs, exact result cache disabled; concurrent Mac validation jobs may contend" } }, { "role": "body", "runtime": "onnx-linux-cuda-w4-block16", "source_checkpoint_sha256": "036be44e87c05759c18297426d2d531b0c3ac04af172886071de2ecdd6703981", "model_sha256": "2cb6c60a23a2e9948b9df9679de0f8b2c849dd81bc54abbf4b23f64148edf574", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9867530173682661, "crops_per_second": 347.07615365168624, "buckets": { "english": { "cases": 50, "character_parity": 0.9991645781119465, "exact_line_parity": 0.94, "candidate_CER": 0.0008356545961002785, "source_CER": 0.0011142061281337048, "CER_delta": -0.0002785515320334262 }, "handwriting": { "cases": 50, "character_parity": 0.9948638931689779, "exact_line_parity": 0.84, "candidate_CER": 0.010230179028132993, "source_CER": 0.010741687979539642, "CER_delta": -0.0005115089514066496 }, "mixed": { "cases": 50, "character_parity": 0.9952095808383233, "exact_line_parity": 0.92, "candidate_CER": 0.010104011887072809, "source_CER": 0.00861812778603269, "CER_delta": 0.0014858841010401188 }, "newspaper": { "cases": 50, "character_parity": 0.9995361781076066, "exact_line_parity": 0.98, "candidate_CER": 0.026901669758812616, "source_CER": 0.026437847866419294, "CER_delta": 0.00046382189239332097 }, "printed": { "cases": 50, "character_parity": 0.9963476990504018, "exact_line_parity": 0.9, "candidate_CER": 0.006559766763848397, "source_CER": 0.00510204081632653, "CER_delta": 0.0014577259475218659 }, "synthetic": { "cases": 50, "character_parity": 0.9867530173682661, "exact_line_parity": 0.78, "candidate_CER": 0.05353564271399943, "source_CER": 0.05410821643286573, "CER_delta": -0.000572573718866304 } }, "execution": { "runtime_version": "1.23.2", "platform": "Linux-7.0.0-31-generic-x86_64-with-glibc2.43", "threads": 2, "batch": 1, "timing_scope": "Preprocessed crop -> compact tokens/confidence; excludes model load, preprocessing, CTC/bidi decode; caches disabled; first shape warmed only", "provider": "CUDAExecutionProvider", "actual_profile_providers": { "CUDAExecutionProvider": 448, "CPUExecutionProvider": 3 }, "baseline_sha256": "c9228d9d65efae590dc9c307f4909cea4d436929cc6887875ff5d2cbd3f7879b", "manifest_sha256": "7e7049a3b56323588ae1f6bd38b1753afc2b1149d275c6f0f07c6993be991978" } }, { "role": "body", "runtime": "tract-cpu-w4-block16", "source_checkpoint_sha256": "036be44e87c05759c18297426d2d531b0c3ac04af172886071de2ecdd6703981", "model_sha256": "2cb6c60a23a2e9948b9df9679de0f8b2c849dd81bc54abbf4b23f64148edf574", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9870473947600824, "crops_per_second": 21.779274798978328, "buckets": { "english": { "cases": 50, "character_parity": 0.9991645781119465, "exact_line_parity": 0.94, "candidate_CER": 0.0008356545961002785, "source_CER": 0.0011142061281337048, "CER_delta": -0.0002785515320334262 }, "handwriting": { "cases": 50, "character_parity": 0.9948638931689779, "exact_line_parity": 0.84, "candidate_CER": 0.010230179028132993, "source_CER": 0.010741687979539642, "CER_delta": -0.0005115089514066496 }, "mixed": { "cases": 50, "character_parity": 0.9952095808383233, "exact_line_parity": 0.92, "candidate_CER": 0.010104011887072809, "source_CER": 0.00861812778603269, "CER_delta": 0.0014858841010401188 }, "newspaper": { "cases": 50, "character_parity": 0.9995361781076066, "exact_line_parity": 0.98, "candidate_CER": 0.026901669758812616, "source_CER": 0.026437847866419294, "CER_delta": 0.00046382189239332097 }, "printed": { "cases": 50, "character_parity": 0.9963476990504018, "exact_line_parity": 0.9, "candidate_CER": 0.006559766763848397, "source_CER": 0.00510204081632653, "CER_delta": 0.0014577259475218659 }, "synthetic": { "cases": 50, "character_parity": 0.9870473947600824, "exact_line_parity": 0.78, "candidate_CER": 0.05382192957343258, "source_CER": 0.05410821643286573, "CER_delta": -0.000286286859433152 } }, "execution": { "timing_scope": "Batch1, native model time after plan warmup; excludes plan preparation, input IO, CTC/bidi; exact-result cache off", "version": "0.23.9-pre", "plan_cache_hits": 296 } }, { "role": "body", "runtime": "onnx-w4-block16", "source_checkpoint_sha256": "036be44e87c05759c18297426d2d531b0c3ac04af172886071de2ecdd6703981", "model_sha256": "2cb6c60a23a2e9948b9df9679de0f8b2c849dd81bc54abbf4b23f64148edf574", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9841036208419194, "crops_per_second": 8.33820322202337, "buckets": { "english": { "cases": 50, "character_parity": 0.998886104149262, "exact_line_parity": 0.92, "candidate_CER": 0.0011142061281337048, "source_CER": 0.0011142061281337048, "CER_delta": 0.0 }, "handwriting": { "cases": 50, "character_parity": 0.9948638931689779, "exact_line_parity": 0.84, "candidate_CER": 0.010230179028132993, "source_CER": 0.010741687979539642, "CER_delta": -0.0005115089514066496 }, "mixed": { "cases": 50, "character_parity": 0.9952095808383233, "exact_line_parity": 0.92, "candidate_CER": 0.010104011887072809, "source_CER": 0.00861812778603269, "CER_delta": 0.0014858841010401188 }, "newspaper": { "cases": 50, "character_parity": 0.9995361781076066, "exact_line_parity": 0.98, "candidate_CER": 0.026901669758812616, "source_CER": 0.026437847866419294, "CER_delta": 0.00046382189239332097 }, "printed": { "cases": 50, "character_parity": 0.9963476990504018, "exact_line_parity": 0.9, "candidate_CER": 0.006559766763848397, "source_CER": 0.00510204081632653, "CER_delta": 0.0014577259475218659 }, "synthetic": { "cases": 50, "character_parity": 0.9841036208419194, "exact_line_parity": 0.78, "candidate_CER": 0.05181792155740052, "source_CER": 0.05410821643286573, "CER_delta": -0.002290294875465216 } }, "execution": { "runtime_version": "1.31.0", "platform": "macOS-26.5.1-arm64-arm-64bit", "threads": 2, "batch": 1, "timing_scope": "Preprocessed crop -> compact tokens/confidence; excludes model load, preprocessing, CTC/bidi decode; caches disabled; first shape warmed only", "provider": "CPUExecutionProvider", "actual_profile_providers": { "CPUExecutionProvider": 370 }, "baseline_sha256": "c9228d9d65efae590dc9c307f4909cea4d436929cc6887875ff5d2cbd3f7879b", "manifest_sha256": "7e7049a3b56323588ae1f6bd38b1753afc2b1149d275c6f0f07c6993be991978" } }, { "role": "body", "runtime": "litert-dynamic4-block32", "source_checkpoint_sha256": "036be44e87c05759c18297426d2d531b0c3ac04af172886071de2ecdd6703981", "model_sha256": "1228832b077ff44b4b79dbac09ef6ca40ae4caf53e0dde13e61d49be5ba19824", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9826317338828378, "crops_per_second": 43.49684545370007, "buckets": { "english": { "cases": 50, "character_parity": 0.9980506822612085, "exact_line_parity": 0.9, "candidate_CER": 0.0019498607242339832, "source_CER": 0.0011142061281337048, "CER_delta": 0.0008356545961002785 }, "handwriting": { "cases": 50, "character_parity": 0.9897277863379559, "exact_line_parity": 0.68, "candidate_CER": 0.014833759590792838, "source_CER": 0.010741687979539642, "CER_delta": 0.004092071611253197 }, "mixed": { "cases": 50, "character_parity": 0.9952095808383233, "exact_line_parity": 0.86, "candidate_CER": 0.008915304606240713, "source_CER": 0.00861812778603269, "CER_delta": 0.00029717682020802375 }, "newspaper": { "cases": 50, "character_parity": 0.9967532467532467, "exact_line_parity": 0.92, "candidate_CER": 0.029684601113172542, "source_CER": 0.026437847866419294, "CER_delta": 0.003246753246753247 }, "printed": { "cases": 50, "character_parity": 0.9934258582907232, "exact_line_parity": 0.84, "candidate_CER": 0.008746355685131196, "source_CER": 0.00510204081632653, "CER_delta": 0.0036443148688046646 }, "synthetic": { "cases": 50, "character_parity": 0.9826317338828378, "exact_line_parity": 0.7, "candidate_CER": 0.06012024048096192, "source_CER": 0.05410821643286573, "CER_delta": 0.006012024048096192 } }, "execution": { "platform": "Linux-7.0.0-31-generic-x86_64-with-glibc2.43", "threads": 2, "batch": 1, "timing_scope": "Preprocessed crop -> compact tokens/confidence; excludes model load, preprocessing, CTC/bidi decode; caches disabled; first shape warmed only", "provider": "CPUExecutionProvider", "baseline_sha256": "c9228d9d65efae590dc9c307f4909cea4d436929cc6887875ff5d2cbd3f7879b", "manifest_sha256": "7e7049a3b56323588ae1f6bd38b1753afc2b1149d275c6f0f07c6993be991978" } }, { "role": "body", "runtime": "tract-metal-w4-block16", "source_checkpoint_sha256": "036be44e87c05759c18297426d2d531b0c3ac04af172886071de2ecdd6703981", "model_sha256": "2cb6c60a23a2e9948b9df9679de0f8b2c849dd81bc54abbf4b23f64148edf574", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9870473947600824, "crops_per_second": 27.301717529508743, "buckets": { "english": { "cases": 50, "character_parity": 0.9991645781119465, "exact_line_parity": 0.94, "candidate_CER": 0.0008356545961002785, "source_CER": 0.0011142061281337048, "CER_delta": -0.0002785515320334262 }, "handwriting": { "cases": 50, "character_parity": 0.9948638931689779, "exact_line_parity": 0.84, "candidate_CER": 0.010230179028132993, "source_CER": 0.010741687979539642, "CER_delta": -0.0005115089514066496 }, "mixed": { "cases": 50, "character_parity": 0.9952095808383233, "exact_line_parity": 0.92, "candidate_CER": 0.010104011887072809, "source_CER": 0.00861812778603269, "CER_delta": 0.0014858841010401188 }, "newspaper": { "cases": 50, "character_parity": 0.9995361781076066, "exact_line_parity": 0.98, "candidate_CER": 0.026901669758812616, "source_CER": 0.026437847866419294, "CER_delta": 0.00046382189239332097 }, "printed": { "cases": 50, "character_parity": 0.9963476990504018, "exact_line_parity": 0.9, "candidate_CER": 0.006559766763848397, "source_CER": 0.00510204081632653, "CER_delta": 0.0014577259475218659 }, "synthetic": { "cases": 50, "character_parity": 0.9870473947600824, "exact_line_parity": 0.78, "candidate_CER": 0.05382192957343258, "source_CER": 0.05410821643286573, "CER_delta": -0.000286286859433152 } }, "execution": { "timing_scope": "Batch1, native model time after plan warmup; excludes plan preparation, input IO, CTC/bidi; exact-result cache off", "version": "0.23.9-pre", "plan_cache_hits": 296 } }, { "role": "numeric_math", "runtime": "browser-wasm", "source_checkpoint_sha256": "6e3869a5cdebff89ce6cfbbcde8ce2d5a1aa9cfe342679d469d5da08bb82b443", "model_sha256": "de7b841b7e1a3ca56da0e76c90f343d8fdad780d464e9b0a30818e71f8a75916", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9870091526424565, "crops_per_second": 0.45353221890262324, "buckets": { "english": { "cases": 50, "character_parity": 0.9991645781119465, "exact_line_parity": 0.94, "candidate_CER": 0.0008356545961002785, "source_CER": 0.0011142061281337048, "CER_delta": -0.0002785515320334262 }, "handwriting": { "cases": 50, "character_parity": 0.9948770491803278, "exact_line_parity": 0.86, "candidate_CER": 0.010230179028132993, "source_CER": 0.009718670076726343, "CER_delta": 0.0005115089514066496 }, "mixed": { "cases": 50, "character_parity": 0.9973005398920216, "exact_line_parity": 0.96, "candidate_CER": 0.009806835066864784, "source_CER": 0.010401188707280832, "CER_delta": -0.0005943536404160475 }, "newspaper": { "cases": 50, "character_parity": 0.9990723562152134, "exact_line_parity": 0.96, "candidate_CER": 0.02736549165120594, "source_CER": 0.026437847866419294, "CER_delta": 0.0009276437847866419 }, "printed": { "cases": 50, "character_parity": 0.9978070175438597, "exact_line_parity": 0.94, "candidate_CER": 0.00510204081632653, "source_CER": 0.0058309037900874635, "CER_delta": -0.0007288629737609329 }, "synthetic": { "cases": 50, "character_parity": 0.9870091526424565, "exact_line_parity": 0.8, "candidate_CER": 0.05553965073003149, "source_CER": 0.05468079015173204, "CER_delta": 0.0008588605782994561 } }, "execution": { "threads": 1, "timing_scope": "Warm bucket, preprocessed crop to compact outputs, exact result cache disabled; concurrent Mac validation jobs may contend" } }, { "role": "numeric_math", "runtime": "onnx-linux-cpu-hqq4-block32", "source_checkpoint_sha256": "6e3869a5cdebff89ce6cfbbcde8ce2d5a1aa9cfe342679d469d5da08bb82b443", "model_sha256": "b8e5b2aded4982446f5ac199de699e52ee04816fc69acfaa05b65cd7e0977d02", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9858281665190434, "crops_per_second": 13.449897637871674, "buckets": { "english": { "cases": 50, "character_parity": 0.999443052074631, "exact_line_parity": 0.96, "candidate_CER": 0.0011142061281337048, "source_CER": 0.0011142061281337048, "CER_delta": 0.0 }, "handwriting": { "cases": 50, "character_parity": 0.9938524590163934, "exact_line_parity": 0.84, "candidate_CER": 0.010230179028132993, "source_CER": 0.009718670076726343, "CER_delta": 0.0005115089514066496 }, "mixed": { "cases": 50, "character_parity": 0.997000599880024, "exact_line_parity": 0.96, "candidate_CER": 0.011589895988112928, "source_CER": 0.010401188707280832, "CER_delta": 0.001188707280832095 }, "newspaper": { "cases": 50, "character_parity": 0.9995361781076066, "exact_line_parity": 0.98, "candidate_CER": 0.026901669758812616, "source_CER": 0.026437847866419294, "CER_delta": 0.00046382189239332097 }, "printed": { "cases": 50, "character_parity": 0.9992690058479532, "exact_line_parity": 0.98, "candidate_CER": 0.006559766763848397, "source_CER": 0.0058309037900874635, "CER_delta": 0.0007288629737609329 }, "synthetic": { "cases": 50, "character_parity": 0.9858281665190434, "exact_line_parity": 0.76, "candidate_CER": 0.0572573718866304, "source_CER": 0.05468079015173204, "CER_delta": 0.002576581734898368 } }, "execution": { "runtime_version": "1.31.0", "platform": "Linux-7.0.0-31-generic-x86_64-with-glibc2.43", "threads": 2, "batch": 1, "timing_scope": "Preprocessed crop -> compact tokens/confidence; excludes model load, preprocessing, CTC/bidi decode; caches disabled; first shape warmed only", "provider": "CPUExecutionProvider", "actual_profile_providers": { "CPUExecutionProvider": 371 }, "baseline_sha256": "39c044c875b75d0082e35d2624a617f8320a95db4f3c3c861dbe7f76099815f9", "manifest_sha256": "7e7049a3b56323588ae1f6bd38b1753afc2b1149d275c6f0f07c6993be991978" } }, { "role": "numeric_math", "runtime": "coreml-w4-block16", "source_checkpoint_sha256": "6e3869a5cdebff89ce6cfbbcde8ce2d5a1aa9cfe342679d469d5da08bb82b443", "model_sha256": null, "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9852376734573369, "crops_per_second": 47.452310161828514, "buckets": { "english": { "cases": 50, "character_parity": 0.998886104149262, "exact_line_parity": 0.94, "candidate_CER": 0.001671309192200557, "source_CER": 0.0011142061281337048, "CER_delta": 0.0005571030640668524 }, "handwriting": { "cases": 50, "character_parity": 0.9923155737704918, "exact_line_parity": 0.76, "candidate_CER": 0.011764705882352941, "source_CER": 0.009718670076726343, "CER_delta": 0.0020460358056265983 }, "mixed": { "cases": 50, "character_parity": 0.9961007798440312, "exact_line_parity": 0.9, "candidate_CER": 0.010104011887072809, "source_CER": 0.010401188707280832, "CER_delta": -0.00029717682020802375 }, "newspaper": { "cases": 50, "character_parity": 0.99860853432282, "exact_line_parity": 0.94, "candidate_CER": 0.027829313543599257, "source_CER": 0.026437847866419294, "CER_delta": 0.0013914656771799629 }, "printed": { "cases": 50, "character_parity": 0.9978070175438597, "exact_line_parity": 0.96, "candidate_CER": 0.00510204081632653, "source_CER": 0.0058309037900874635, "CER_delta": -0.0007288629737609329 }, "synthetic": { "cases": 50, "character_parity": 0.9852376734573369, "exact_line_parity": 0.78, "candidate_CER": 0.056112224448897796, "source_CER": 0.05468079015173204, "CER_delta": 0.0014314342971657602 } }, "execution": { "platform": "macOS-26.5.1-arm64-arm-64bit", "threads": 2, "batch": 1, "timing_scope": "Preprocessed crop -> compact tokens/confidence; excludes model load, preprocessing, CTC/bidi decode; caches disabled; first shape warmed only", "provider": "CPUExecutionProvider", "baseline_sha256": "39c044c875b75d0082e35d2624a617f8320a95db4f3c3c861dbe7f76099815f9", "manifest_sha256": "7e7049a3b56323588ae1f6bd38b1753afc2b1149d275c6f0f07c6993be991978" } }, { "role": "numeric_math", "runtime": "onnx-hqq4-block32", "source_checkpoint_sha256": "6e3869a5cdebff89ce6cfbbcde8ce2d5a1aa9cfe342679d469d5da08bb82b443", "model_sha256": "b8e5b2aded4982446f5ac199de699e52ee04816fc69acfaa05b65cd7e0977d02", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9858281665190434, "crops_per_second": 5.269215760371242, "buckets": { "english": { "cases": 50, "character_parity": 0.999443052074631, "exact_line_parity": 0.96, "candidate_CER": 0.0011142061281337048, "source_CER": 0.0011142061281337048, "CER_delta": 0.0 }, "handwriting": { "cases": 50, "character_parity": 0.9938524590163934, "exact_line_parity": 0.84, "candidate_CER": 0.010230179028132993, "source_CER": 0.009718670076726343, "CER_delta": 0.0005115089514066496 }, "mixed": { "cases": 50, "character_parity": 0.997000599880024, "exact_line_parity": 0.96, "candidate_CER": 0.011589895988112928, "source_CER": 0.010401188707280832, "CER_delta": 0.001188707280832095 }, "newspaper": { "cases": 50, "character_parity": 0.9995361781076066, "exact_line_parity": 0.98, "candidate_CER": 0.026901669758812616, "source_CER": 0.026437847866419294, "CER_delta": 0.00046382189239332097 }, "printed": { "cases": 50, "character_parity": 0.9992690058479532, "exact_line_parity": 0.98, "candidate_CER": 0.006559766763848397, "source_CER": 0.0058309037900874635, "CER_delta": 0.0007288629737609329 }, "synthetic": { "cases": 50, "character_parity": 0.9858281665190434, "exact_line_parity": 0.76, "candidate_CER": 0.0572573718866304, "source_CER": 0.05468079015173204, "CER_delta": 0.002576581734898368 } }, "execution": { "runtime_version": "1.31.0", "platform": "macOS-26.5.1-arm64-arm-64bit", "threads": 2, "batch": 1, "timing_scope": "Preprocessed crop -> compact tokens/confidence; excludes model load, preprocessing, CTC/bidi decode; caches disabled; first shape warmed only", "provider": "CPUExecutionProvider", "actual_profile_providers": { "CPUExecutionProvider": 370 }, "baseline_sha256": "39c044c875b75d0082e35d2624a617f8320a95db4f3c3c861dbe7f76099815f9", "manifest_sha256": "7e7049a3b56323588ae1f6bd38b1753afc2b1149d275c6f0f07c6993be991978" } }, { "role": "numeric_math", "runtime": "tract-linux-cuda-w4-block16", "source_checkpoint_sha256": "6e3869a5cdebff89ce6cfbbcde8ce2d5a1aa9cfe342679d469d5da08bb82b443", "model_sha256": "de7b841b7e1a3ca56da0e76c90f343d8fdad780d464e9b0a30818e71f8a75916", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9870091526424565, "crops_per_second": 222.39857308838302, "buckets": { "english": { "cases": 50, "character_parity": 0.9991645781119465, "exact_line_parity": 0.94, "candidate_CER": 0.0008356545961002785, "source_CER": 0.0011142061281337048, "CER_delta": -0.0002785515320334262 }, "handwriting": { "cases": 50, "character_parity": 0.9948770491803278, "exact_line_parity": 0.86, "candidate_CER": 0.010230179028132993, "source_CER": 0.009718670076726343, "CER_delta": 0.0005115089514066496 }, "mixed": { "cases": 50, "character_parity": 0.9973005398920216, "exact_line_parity": 0.96, "candidate_CER": 0.009806835066864784, "source_CER": 0.010401188707280832, "CER_delta": -0.0005943536404160475 }, "newspaper": { "cases": 50, "character_parity": 0.9990723562152134, "exact_line_parity": 0.96, "candidate_CER": 0.02736549165120594, "source_CER": 0.026437847866419294, "CER_delta": 0.0009276437847866419 }, "printed": { "cases": 50, "character_parity": 0.9978070175438597, "exact_line_parity": 0.94, "candidate_CER": 0.00510204081632653, "source_CER": 0.0058309037900874635, "CER_delta": -0.0007288629737609329 }, "synthetic": { "cases": 50, "character_parity": 0.9870091526424565, "exact_line_parity": 0.8, "candidate_CER": 0.05553965073003149, "source_CER": 0.05468079015173204, "CER_delta": 0.0008588605782994561 } }, "execution": { "timing_scope": "Batch1, native model time after plan warmup; excludes plan preparation, input IO, CTC/bidi; exact-result cache off", "version": "0.23.9-pre", "plan_cache_hits": 296 } }, { "role": "numeric_math", "runtime": "browser-webgpu", "source_checkpoint_sha256": "6e3869a5cdebff89ce6cfbbcde8ce2d5a1aa9cfe342679d469d5da08bb82b443", "model_sha256": "de7b841b7e1a3ca56da0e76c90f343d8fdad780d464e9b0a30818e71f8a75916", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9870091526424565, "crops_per_second": 7.434653116622879, "buckets": { "english": { "cases": 50, "character_parity": 0.9991645781119465, "exact_line_parity": 0.94, "candidate_CER": 0.0008356545961002785, "source_CER": 0.0011142061281337048, "CER_delta": -0.0002785515320334262 }, "handwriting": { "cases": 50, "character_parity": 0.9948770491803278, "exact_line_parity": 0.86, "candidate_CER": 0.010230179028132993, "source_CER": 0.009718670076726343, "CER_delta": 0.0005115089514066496 }, "mixed": { "cases": 50, "character_parity": 0.9973005398920216, "exact_line_parity": 0.96, "candidate_CER": 0.009806835066864784, "source_CER": 0.010401188707280832, "CER_delta": -0.0005943536404160475 }, "newspaper": { "cases": 50, "character_parity": 0.9990723562152134, "exact_line_parity": 0.96, "candidate_CER": 0.02736549165120594, "source_CER": 0.026437847866419294, "CER_delta": 0.0009276437847866419 }, "printed": { "cases": 50, "character_parity": 0.9978070175438597, "exact_line_parity": 0.94, "candidate_CER": 0.00510204081632653, "source_CER": 0.0058309037900874635, "CER_delta": -0.0007288629737609329 }, "synthetic": { "cases": 50, "character_parity": 0.9870091526424565, "exact_line_parity": 0.8, "candidate_CER": 0.05553965073003149, "source_CER": 0.05468079015173204, "CER_delta": 0.0008588605782994561 } }, "execution": { "threads": 2, "timing_scope": "Warm bucket, preprocessed crop to compact outputs, exact result cache disabled; concurrent Mac validation jobs may contend" } }, { "role": "numeric_math", "runtime": "onnx-linux-cuda-w4-block16", "source_checkpoint_sha256": "6e3869a5cdebff89ce6cfbbcde8ce2d5a1aa9cfe342679d469d5da08bb82b443", "model_sha256": "de7b841b7e1a3ca56da0e76c90f343d8fdad780d464e9b0a30818e71f8a75916", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9873043991733097, "crops_per_second": 330.48427191322395, "buckets": { "english": { "cases": 50, "character_parity": 0.9991645781119465, "exact_line_parity": 0.94, "candidate_CER": 0.0008356545961002785, "source_CER": 0.0011142061281337048, "CER_delta": -0.0002785515320334262 }, "handwriting": { "cases": 50, "character_parity": 0.9948770491803278, "exact_line_parity": 0.86, "candidate_CER": 0.010230179028132993, "source_CER": 0.009718670076726343, "CER_delta": 0.0005115089514066496 }, "mixed": { "cases": 50, "character_parity": 0.9973005398920216, "exact_line_parity": 0.96, "candidate_CER": 0.009806835066864784, "source_CER": 0.010401188707280832, "CER_delta": -0.0005943536404160475 }, "newspaper": { "cases": 50, "character_parity": 0.9990723562152134, "exact_line_parity": 0.96, "candidate_CER": 0.02736549165120594, "source_CER": 0.026437847866419294, "CER_delta": 0.0009276437847866419 }, "printed": { "cases": 50, "character_parity": 0.9978070175438597, "exact_line_parity": 0.94, "candidate_CER": 0.00510204081632653, "source_CER": 0.0058309037900874635, "CER_delta": -0.0007288629737609329 }, "synthetic": { "cases": 50, "character_parity": 0.9873043991733097, "exact_line_parity": 0.8, "candidate_CER": 0.055253363870598336, "source_CER": 0.05468079015173204, "CER_delta": 0.000572573718866304 } }, "execution": { "runtime_version": "1.23.2", "platform": "Linux-7.0.0-31-generic-x86_64-with-glibc2.43", "threads": 2, "batch": 1, "timing_scope": "Preprocessed crop -> compact tokens/confidence; excludes model load, preprocessing, CTC/bidi decode; caches disabled; first shape warmed only", "provider": "CUDAExecutionProvider", "actual_profile_providers": { "CUDAExecutionProvider": 448, "CPUExecutionProvider": 3 }, "baseline_sha256": "39c044c875b75d0082e35d2624a617f8320a95db4f3c3c861dbe7f76099815f9", "manifest_sha256": "7e7049a3b56323588ae1f6bd38b1753afc2b1149d275c6f0f07c6993be991978" } }, { "role": "numeric_math", "runtime": "tract-cpu-w4-block16", "source_checkpoint_sha256": "6e3869a5cdebff89ce6cfbbcde8ce2d5a1aa9cfe342679d469d5da08bb82b443", "model_sha256": "de7b841b7e1a3ca56da0e76c90f343d8fdad780d464e9b0a30818e71f8a75916", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9870091526424565, "crops_per_second": 17.48098874356552, "buckets": { "english": { "cases": 50, "character_parity": 0.9991645781119465, "exact_line_parity": 0.94, "candidate_CER": 0.0008356545961002785, "source_CER": 0.0011142061281337048, "CER_delta": -0.0002785515320334262 }, "handwriting": { "cases": 50, "character_parity": 0.9948770491803278, "exact_line_parity": 0.86, "candidate_CER": 0.010230179028132993, "source_CER": 0.009718670076726343, "CER_delta": 0.0005115089514066496 }, "mixed": { "cases": 50, "character_parity": 0.9973005398920216, "exact_line_parity": 0.96, "candidate_CER": 0.009806835066864784, "source_CER": 0.010401188707280832, "CER_delta": -0.0005943536404160475 }, "newspaper": { "cases": 50, "character_parity": 0.9990723562152134, "exact_line_parity": 0.96, "candidate_CER": 0.02736549165120594, "source_CER": 0.026437847866419294, "CER_delta": 0.0009276437847866419 }, "printed": { "cases": 50, "character_parity": 0.9978070175438597, "exact_line_parity": 0.94, "candidate_CER": 0.00510204081632653, "source_CER": 0.0058309037900874635, "CER_delta": -0.0007288629737609329 }, "synthetic": { "cases": 50, "character_parity": 0.9870091526424565, "exact_line_parity": 0.8, "candidate_CER": 0.05553965073003149, "source_CER": 0.05468079015173204, "CER_delta": 0.0008588605782994561 } }, "execution": { "timing_scope": "Batch1, native model time after plan warmup; excludes plan preparation, input IO, CTC/bidi; exact-result cache off", "version": "0.23.9-pre", "plan_cache_hits": 296 } }, { "role": "numeric_math", "runtime": "onnx-w4-block16", "source_checkpoint_sha256": "6e3869a5cdebff89ce6cfbbcde8ce2d5a1aa9cfe342679d469d5da08bb82b443", "model_sha256": "de7b841b7e1a3ca56da0e76c90f343d8fdad780d464e9b0a30818e71f8a75916", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9861234130498967, "crops_per_second": 5.231096362878766, "buckets": { "english": { "cases": 50, "character_parity": 0.999443052074631, "exact_line_parity": 0.96, "candidate_CER": 0.0005571030640668524, "source_CER": 0.0011142061281337048, "CER_delta": -0.0005571030640668524 }, "handwriting": { "cases": 50, "character_parity": 0.9948770491803278, "exact_line_parity": 0.84, "candidate_CER": 0.010741687979539642, "source_CER": 0.009718670076726343, "CER_delta": 0.0010230179028132991 }, "mixed": { "cases": 50, "character_parity": 0.9973005398920216, "exact_line_parity": 0.96, "candidate_CER": 0.010104011887072809, "source_CER": 0.010401188707280832, "CER_delta": -0.00029717682020802375 }, "newspaper": { "cases": 50, "character_parity": 0.9990723562152134, "exact_line_parity": 0.96, "candidate_CER": 0.02736549165120594, "source_CER": 0.026437847866419294, "CER_delta": 0.0009276437847866419 }, "printed": { "cases": 50, "character_parity": 0.9978070175438597, "exact_line_parity": 0.94, "candidate_CER": 0.00510204081632653, "source_CER": 0.0058309037900874635, "CER_delta": -0.0007288629737609329 }, "synthetic": { "cases": 50, "character_parity": 0.9861234130498967, "exact_line_parity": 0.8, "candidate_CER": 0.052963068995133124, "source_CER": 0.05468079015173204, "CER_delta": -0.0017177211565989122 } }, "execution": { "runtime_version": "1.31.0", "platform": "macOS-26.5.1-arm64-arm-64bit", "threads": 2, "batch": 1, "timing_scope": "Preprocessed crop -> compact tokens/confidence; excludes model load, preprocessing, CTC/bidi decode; caches disabled; first shape warmed only", "provider": "CPUExecutionProvider", "actual_profile_providers": { "CPUExecutionProvider": 370 }, "baseline_sha256": "39c044c875b75d0082e35d2624a617f8320a95db4f3c3c861dbe7f76099815f9", "manifest_sha256": "7e7049a3b56323588ae1f6bd38b1753afc2b1149d275c6f0f07c6993be991978" } }, { "role": "numeric_math", "runtime": "litert-dynamic4-block32", "source_checkpoint_sha256": "6e3869a5cdebff89ce6cfbbcde8ce2d5a1aa9cfe342679d469d5da08bb82b443", "model_sha256": "e0908804c688599e5c42144cf83ba436e8a5fd33e2531b8f3946fc596b8fce17", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.984351933864777, "crops_per_second": 42.00890459026271, "buckets": { "english": { "cases": 50, "character_parity": 0.9974937343358397, "exact_line_parity": 0.86, "candidate_CER": 0.0025069637883008357, "source_CER": 0.0011142061281337048, "CER_delta": 0.001392757660167131 }, "handwriting": { "cases": 50, "character_parity": 0.9902663934426229, "exact_line_parity": 0.76, "candidate_CER": 0.01381074168797954, "source_CER": 0.009718670076726343, "CER_delta": 0.004092071611253197 }, "mixed": { "cases": 50, "character_parity": 0.997000599880024, "exact_line_parity": 0.9, "candidate_CER": 0.011292719167904903, "source_CER": 0.010401188707280832, "CER_delta": 0.0008915304606240713 }, "newspaper": { "cases": 50, "character_parity": 0.99860853432282, "exact_line_parity": 0.96, "candidate_CER": 0.027829313543599257, "source_CER": 0.026437847866419294, "CER_delta": 0.0013914656771799629 }, "printed": { "cases": 50, "character_parity": 0.9956140350877193, "exact_line_parity": 0.9, "candidate_CER": 0.008746355685131196, "source_CER": 0.0058309037900874635, "CER_delta": 0.0029154518950437317 }, "synthetic": { "cases": 50, "character_parity": 0.984351933864777, "exact_line_parity": 0.66, "candidate_CER": 0.059833953621528775, "source_CER": 0.05468079015173204, "CER_delta": 0.005153163469796736 } }, "execution": { "platform": "Linux-7.0.0-31-generic-x86_64-with-glibc2.43", "threads": 2, "batch": 1, "timing_scope": "Preprocessed crop -> compact tokens/confidence; excludes model load, preprocessing, CTC/bidi decode; caches disabled; first shape warmed only", "provider": "CPUExecutionProvider", "baseline_sha256": "39c044c875b75d0082e35d2624a617f8320a95db4f3c3c861dbe7f76099815f9", "manifest_sha256": "7e7049a3b56323588ae1f6bd38b1753afc2b1149d275c6f0f07c6993be991978" } }, { "role": "numeric_math", "runtime": "tract-metal-w4-block16", "source_checkpoint_sha256": "6e3869a5cdebff89ce6cfbbcde8ce2d5a1aa9cfe342679d469d5da08bb82b443", "model_sha256": "de7b841b7e1a3ca56da0e76c90f343d8fdad780d464e9b0a30818e71f8a75916", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9870091526424565, "crops_per_second": 20.460208750116568, "buckets": { "english": { "cases": 50, "character_parity": 0.9991645781119465, "exact_line_parity": 0.94, "candidate_CER": 0.0008356545961002785, "source_CER": 0.0011142061281337048, "CER_delta": -0.0002785515320334262 }, "handwriting": { "cases": 50, "character_parity": 0.9948770491803278, "exact_line_parity": 0.86, "candidate_CER": 0.010230179028132993, "source_CER": 0.009718670076726343, "CER_delta": 0.0005115089514066496 }, "mixed": { "cases": 50, "character_parity": 0.9973005398920216, "exact_line_parity": 0.96, "candidate_CER": 0.009806835066864784, "source_CER": 0.010401188707280832, "CER_delta": -0.0005943536404160475 }, "newspaper": { "cases": 50, "character_parity": 0.9990723562152134, "exact_line_parity": 0.96, "candidate_CER": 0.02736549165120594, "source_CER": 0.026437847866419294, "CER_delta": 0.0009276437847866419 }, "printed": { "cases": 50, "character_parity": 0.9978070175438597, "exact_line_parity": 0.94, "candidate_CER": 0.00510204081632653, "source_CER": 0.0058309037900874635, "CER_delta": -0.0007288629737609329 }, "synthetic": { "cases": 50, "character_parity": 0.9870091526424565, "exact_line_parity": 0.8, "candidate_CER": 0.05553965073003149, "source_CER": 0.05468079015173204, "CER_delta": 0.0008588605782994561 } }, "execution": { "timing_scope": "Batch1, native model time after plan warmup; excludes plan preparation, input IO, CTC/bidi; exact-result cache off", "version": "0.23.9-pre", "plan_cache_hits": 296 } }, { "role": "printed", "runtime": "browser-wasm", "source_checkpoint_sha256": "28c4ca8f9fbbc8c578bb130d997598d2e9c2c4a2dad0b22cb5cc06818ecb988a", "model_sha256": "7f4d08254381c298ce8b88966c4f5b905c7a57baa6f4f8aaf170986874dfc477", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9867178276269185, "crops_per_second": 0.5527785853213841, "buckets": { "english": { "cases": 50, "character_parity": 0.9986076301865776, "exact_line_parity": 0.9, "candidate_CER": 0.001392757660167131, "source_CER": 0.0011142061281337048, "CER_delta": 0.0002785515320334262 }, "handwriting": { "cases": 50, "character_parity": 0.9953775038520801, "exact_line_parity": 0.86, "candidate_CER": 0.011764705882352941, "source_CER": 0.011253196930946292, "CER_delta": 0.0005115089514066496 }, "mixed": { "cases": 50, "character_parity": 0.9958096378329842, "exact_line_parity": 0.92, "candidate_CER": 0.010698365527488856, "source_CER": 0.008023774145616641, "CER_delta": 0.002674591381872214 }, "newspaper": { "cases": 50, "character_parity": 0.9986078886310905, "exact_line_parity": 0.96, "candidate_CER": 0.0287569573283859, "source_CER": 0.02829313543599258, "CER_delta": 0.00046382189239332097 }, "printed": { "cases": 50, "character_parity": 0.9970760233918129, "exact_line_parity": 0.92, "candidate_CER": 0.0058309037900874635, "source_CER": 0.004373177842565598, "CER_delta": 0.0014577259475218659 }, "synthetic": { "cases": 50, "character_parity": 0.9867178276269185, "exact_line_parity": 0.8, "candidate_CER": 0.05496707701116519, "source_CER": 0.0566847981677641, "CER_delta": -0.0017177211565989122 } }, "execution": { "threads": 1, "timing_scope": "Warm bucket, preprocessed crop to compact outputs, exact result cache disabled; concurrent Mac validation jobs may contend" } }, { "role": "printed", "runtime": "onnx-linux-cpu-hqq4-block32", "source_checkpoint_sha256": "28c4ca8f9fbbc8c578bb130d997598d2e9c2c4a2dad0b22cb5cc06818ecb988a", "model_sha256": "8c687c2e7bc56c1fc0878d9eddf86b83cac554764a227cc2f9a32bb9c82b73f2", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9878984651711924, "crops_per_second": 10.156680017912882, "buckets": { "english": { "cases": 50, "character_parity": 0.999443052074631, "exact_line_parity": 0.96, "candidate_CER": 0.0011142061281337048, "source_CER": 0.0011142061281337048, "CER_delta": 0.0 }, "handwriting": { "cases": 50, "character_parity": 0.9958911145351823, "exact_line_parity": 0.88, "candidate_CER": 0.012276214833759591, "source_CER": 0.011253196930946292, "CER_delta": 0.0010230179028132991 }, "mixed": { "cases": 50, "character_parity": 0.9949117030829093, "exact_line_parity": 0.92, "candidate_CER": 0.010698365527488856, "source_CER": 0.008023774145616641, "CER_delta": 0.002674591381872214 }, "newspaper": { "cases": 50, "character_parity": 0.9990719257540603, "exact_line_parity": 0.96, "candidate_CER": 0.02922077922077922, "source_CER": 0.02829313543599258, "CER_delta": 0.0009276437847866419 }, "printed": { "cases": 50, "character_parity": 0.9978070175438597, "exact_line_parity": 0.94, "candidate_CER": 0.0058309037900874635, "source_CER": 0.004373177842565598, "CER_delta": 0.0014577259475218659 }, "synthetic": { "cases": 50, "character_parity": 0.9878984651711924, "exact_line_parity": 0.78, "candidate_CER": 0.0566847981677641, "source_CER": 0.0566847981677641, "CER_delta": 0.0 } }, "execution": { "runtime_version": "1.31.0", "platform": "Linux-7.0.0-31-generic-x86_64-with-glibc2.43", "threads": 2, "batch": 1, "timing_scope": "Preprocessed crop -> compact tokens/confidence; excludes model load, preprocessing, CTC/bidi decode; caches disabled; first shape warmed only", "provider": "CPUExecutionProvider", "actual_profile_providers": { "CPUExecutionProvider": 371 }, "baseline_sha256": "ae72ecf55cff6414f7a006b15cbbd310f6ef021d1a6adb2fb897660b29332b25", "manifest_sha256": "7e7049a3b56323588ae1f6bd38b1753afc2b1149d275c6f0f07c6993be991978" } }, { "role": "printed", "runtime": "coreml-w4-block16", "source_checkpoint_sha256": "28c4ca8f9fbbc8c578bb130d997598d2e9c2c4a2dad0b22cb5cc06818ecb988a", "model_sha256": null, "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.98642266824085, "crops_per_second": 37.901162275606005, "buckets": { "english": { "cases": 50, "character_parity": 0.9991645781119465, "exact_line_parity": 0.94, "candidate_CER": 0.001392757660167131, "source_CER": 0.0011142061281337048, "CER_delta": 0.0002785515320334262 }, "handwriting": { "cases": 50, "character_parity": 0.9933230611196713, "exact_line_parity": 0.76, "candidate_CER": 0.014833759590792838, "source_CER": 0.011253196930946292, "CER_delta": 0.0035805626598465474 }, "mixed": { "cases": 50, "character_parity": 0.9964082609997007, "exact_line_parity": 0.9, "candidate_CER": 0.009212481426448737, "source_CER": 0.008023774145616641, "CER_delta": 0.001188707280832095 }, "newspaper": { "cases": 50, "character_parity": 0.9995359628770302, "exact_line_parity": 0.98, "candidate_CER": 0.0287569573283859, "source_CER": 0.02829313543599258, "CER_delta": 0.00046382189239332097 }, "printed": { "cases": 50, "character_parity": 0.9948830409356725, "exact_line_parity": 0.88, "candidate_CER": 0.006559766763848397, "source_CER": 0.004373177842565598, "CER_delta": 0.002186588921282799 }, "synthetic": { "cases": 50, "character_parity": 0.98642266824085, "exact_line_parity": 0.82, "candidate_CER": 0.05553965073003149, "source_CER": 0.0566847981677641, "CER_delta": -0.001145147437732608 } }, "execution": { "platform": "macOS-26.5.1-arm64-arm-64bit", "threads": 2, "batch": 1, "timing_scope": "Preprocessed crop -> compact tokens/confidence; excludes model load, preprocessing, CTC/bidi decode; caches disabled; first shape warmed only", "provider": "CPUExecutionProvider", "baseline_sha256": "ae72ecf55cff6414f7a006b15cbbd310f6ef021d1a6adb2fb897660b29332b25", "manifest_sha256": "7e7049a3b56323588ae1f6bd38b1753afc2b1149d275c6f0f07c6993be991978" } }, { "role": "printed", "runtime": "onnx-hqq4-block32", "source_checkpoint_sha256": "28c4ca8f9fbbc8c578bb130d997598d2e9c2c4a2dad0b22cb5cc06818ecb988a", "model_sha256": "8c687c2e7bc56c1fc0878d9eddf86b83cac554764a227cc2f9a32bb9c82b73f2", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9878984651711924, "crops_per_second": 5.344448209616115, "buckets": { "english": { "cases": 50, "character_parity": 0.999443052074631, "exact_line_parity": 0.96, "candidate_CER": 0.0011142061281337048, "source_CER": 0.0011142061281337048, "CER_delta": 0.0 }, "handwriting": { "cases": 50, "character_parity": 0.9958911145351823, "exact_line_parity": 0.88, "candidate_CER": 0.012276214833759591, "source_CER": 0.011253196930946292, "CER_delta": 0.0010230179028132991 }, "mixed": { "cases": 50, "character_parity": 0.9949117030829093, "exact_line_parity": 0.92, "candidate_CER": 0.010698365527488856, "source_CER": 0.008023774145616641, "CER_delta": 0.002674591381872214 }, "newspaper": { "cases": 50, "character_parity": 0.9990719257540603, "exact_line_parity": 0.96, "candidate_CER": 0.02922077922077922, "source_CER": 0.02829313543599258, "CER_delta": 0.0009276437847866419 }, "printed": { "cases": 50, "character_parity": 0.9978070175438597, "exact_line_parity": 0.94, "candidate_CER": 0.0058309037900874635, "source_CER": 0.004373177842565598, "CER_delta": 0.0014577259475218659 }, "synthetic": { "cases": 50, "character_parity": 0.9878984651711924, "exact_line_parity": 0.78, "candidate_CER": 0.0566847981677641, "source_CER": 0.0566847981677641, "CER_delta": 0.0 } }, "execution": { "runtime_version": "1.31.0", "platform": "macOS-26.5.1-arm64-arm-64bit", "threads": 2, "batch": 1, "timing_scope": "Preprocessed crop -> compact tokens/confidence; excludes model load, preprocessing, CTC/bidi decode; caches disabled; first shape warmed only", "provider": "CPUExecutionProvider", "actual_profile_providers": { "CPUExecutionProvider": 370 }, "baseline_sha256": "ae72ecf55cff6414f7a006b15cbbd310f6ef021d1a6adb2fb897660b29332b25", "manifest_sha256": "7e7049a3b56323588ae1f6bd38b1753afc2b1149d275c6f0f07c6993be991978" } }, { "role": "printed", "runtime": "tract-linux-cuda-w4-block16", "source_checkpoint_sha256": "28c4ca8f9fbbc8c578bb130d997598d2e9c2c4a2dad0b22cb5cc06818ecb988a", "model_sha256": "7f4d08254381c298ce8b88966c4f5b905c7a57baa6f4f8aaf170986874dfc477", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9867178276269185, "crops_per_second": 225.45710142207844, "buckets": { "english": { "cases": 50, "character_parity": 0.9986076301865776, "exact_line_parity": 0.9, "candidate_CER": 0.001392757660167131, "source_CER": 0.0011142061281337048, "CER_delta": 0.0002785515320334262 }, "handwriting": { "cases": 50, "character_parity": 0.9953775038520801, "exact_line_parity": 0.86, "candidate_CER": 0.011764705882352941, "source_CER": 0.011253196930946292, "CER_delta": 0.0005115089514066496 }, "mixed": { "cases": 50, "character_parity": 0.9958096378329842, "exact_line_parity": 0.92, "candidate_CER": 0.010698365527488856, "source_CER": 0.008023774145616641, "CER_delta": 0.002674591381872214 }, "newspaper": { "cases": 50, "character_parity": 0.9986078886310905, "exact_line_parity": 0.96, "candidate_CER": 0.0287569573283859, "source_CER": 0.02829313543599258, "CER_delta": 0.00046382189239332097 }, "printed": { "cases": 50, "character_parity": 0.9970760233918129, "exact_line_parity": 0.92, "candidate_CER": 0.0058309037900874635, "source_CER": 0.004373177842565598, "CER_delta": 0.0014577259475218659 }, "synthetic": { "cases": 50, "character_parity": 0.9867178276269185, "exact_line_parity": 0.8, "candidate_CER": 0.05496707701116519, "source_CER": 0.0566847981677641, "CER_delta": -0.0017177211565989122 } }, "execution": { "timing_scope": "Batch1, native model time after plan warmup; excludes plan preparation, input IO, CTC/bidi; exact-result cache off", "version": "0.23.9-pre", "plan_cache_hits": 296 } }, { "role": "printed", "runtime": "browser-webgpu", "source_checkpoint_sha256": "28c4ca8f9fbbc8c578bb130d997598d2e9c2c4a2dad0b22cb5cc06818ecb988a", "model_sha256": "7f4d08254381c298ce8b88966c4f5b905c7a57baa6f4f8aaf170986874dfc477", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9867178276269185, "crops_per_second": 8.61619338907013, "buckets": { "english": { "cases": 50, "character_parity": 0.9986076301865776, "exact_line_parity": 0.9, "candidate_CER": 0.001392757660167131, "source_CER": 0.0011142061281337048, "CER_delta": 0.0002785515320334262 }, "handwriting": { "cases": 50, "character_parity": 0.9953775038520801, "exact_line_parity": 0.86, "candidate_CER": 0.011764705882352941, "source_CER": 0.011253196930946292, "CER_delta": 0.0005115089514066496 }, "mixed": { "cases": 50, "character_parity": 0.9958096378329842, "exact_line_parity": 0.92, "candidate_CER": 0.010698365527488856, "source_CER": 0.008023774145616641, "CER_delta": 0.002674591381872214 }, "newspaper": { "cases": 50, "character_parity": 0.9986078886310905, "exact_line_parity": 0.96, "candidate_CER": 0.0287569573283859, "source_CER": 0.02829313543599258, "CER_delta": 0.00046382189239332097 }, "printed": { "cases": 50, "character_parity": 0.9970760233918129, "exact_line_parity": 0.92, "candidate_CER": 0.0058309037900874635, "source_CER": 0.004373177842565598, "CER_delta": 0.0014577259475218659 }, "synthetic": { "cases": 50, "character_parity": 0.9867178276269185, "exact_line_parity": 0.8, "candidate_CER": 0.05496707701116519, "source_CER": 0.0566847981677641, "CER_delta": -0.0017177211565989122 } }, "execution": { "threads": 2, "timing_scope": "Warm bucket, preprocessed crop to compact outputs, exact result cache disabled; concurrent Mac validation jobs may contend" } }, { "role": "printed", "runtime": "onnx-linux-cuda-w4-block16", "source_checkpoint_sha256": "28c4ca8f9fbbc8c578bb130d997598d2e9c2c4a2dad0b22cb5cc06818ecb988a", "model_sha256": "7f4d08254381c298ce8b88966c4f5b905c7a57baa6f4f8aaf170986874dfc477", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.98642266824085, "crops_per_second": 315.415471898614, "buckets": { "english": { "cases": 50, "character_parity": 0.9986076301865776, "exact_line_parity": 0.9, "candidate_CER": 0.001392757660167131, "source_CER": 0.0011142061281337048, "CER_delta": 0.0002785515320334262 }, "handwriting": { "cases": 50, "character_parity": 0.9953775038520801, "exact_line_parity": 0.86, "candidate_CER": 0.011764705882352941, "source_CER": 0.011253196930946292, "CER_delta": 0.0005115089514066496 }, "mixed": { "cases": 50, "character_parity": 0.9958096378329842, "exact_line_parity": 0.92, "candidate_CER": 0.010698365527488856, "source_CER": 0.008023774145616641, "CER_delta": 0.002674591381872214 }, "newspaper": { "cases": 50, "character_parity": 0.9986078886310905, "exact_line_parity": 0.96, "candidate_CER": 0.0287569573283859, "source_CER": 0.02829313543599258, "CER_delta": 0.00046382189239332097 }, "printed": { "cases": 50, "character_parity": 0.9970760233918129, "exact_line_parity": 0.92, "candidate_CER": 0.0058309037900874635, "source_CER": 0.004373177842565598, "CER_delta": 0.0014577259475218659 }, "synthetic": { "cases": 50, "character_parity": 0.98642266824085, "exact_line_parity": 0.8, "candidate_CER": 0.055253363870598336, "source_CER": 0.0566847981677641, "CER_delta": -0.0014314342971657602 } }, "execution": { "runtime_version": "1.23.2", "platform": "Linux-7.0.0-31-generic-x86_64-with-glibc2.43", "threads": 2, "batch": 1, "timing_scope": "Preprocessed crop -> compact tokens/confidence; excludes model load, preprocessing, CTC/bidi decode; caches disabled; first shape warmed only", "provider": "CUDAExecutionProvider", "actual_profile_providers": { "CUDAExecutionProvider": 448, "CPUExecutionProvider": 3 }, "baseline_sha256": "ae72ecf55cff6414f7a006b15cbbd310f6ef021d1a6adb2fb897660b29332b25", "manifest_sha256": "7e7049a3b56323588ae1f6bd38b1753afc2b1149d275c6f0f07c6993be991978" } }, { "role": "printed", "runtime": "tract-cpu-w4-block16", "source_checkpoint_sha256": "28c4ca8f9fbbc8c578bb130d997598d2e9c2c4a2dad0b22cb5cc06818ecb988a", "model_sha256": "7f4d08254381c298ce8b88966c4f5b905c7a57baa6f4f8aaf170986874dfc477", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9867178276269185, "crops_per_second": 17.84714031262302, "buckets": { "english": { "cases": 50, "character_parity": 0.9986076301865776, "exact_line_parity": 0.9, "candidate_CER": 0.001392757660167131, "source_CER": 0.0011142061281337048, "CER_delta": 0.0002785515320334262 }, "handwriting": { "cases": 50, "character_parity": 0.9953775038520801, "exact_line_parity": 0.86, "candidate_CER": 0.011764705882352941, "source_CER": 0.011253196930946292, "CER_delta": 0.0005115089514066496 }, "mixed": { "cases": 50, "character_parity": 0.9958096378329842, "exact_line_parity": 0.92, "candidate_CER": 0.010698365527488856, "source_CER": 0.008023774145616641, "CER_delta": 0.002674591381872214 }, "newspaper": { "cases": 50, "character_parity": 0.9986078886310905, "exact_line_parity": 0.96, "candidate_CER": 0.0287569573283859, "source_CER": 0.02829313543599258, "CER_delta": 0.00046382189239332097 }, "printed": { "cases": 50, "character_parity": 0.9970760233918129, "exact_line_parity": 0.92, "candidate_CER": 0.0058309037900874635, "source_CER": 0.004373177842565598, "CER_delta": 0.0014577259475218659 }, "synthetic": { "cases": 50, "character_parity": 0.9867178276269185, "exact_line_parity": 0.8, "candidate_CER": 0.05496707701116519, "source_CER": 0.0566847981677641, "CER_delta": -0.0017177211565989122 } }, "execution": { "timing_scope": "Batch1, native model time after plan warmup; excludes plan preparation, input IO, CTC/bidi; exact-result cache off", "version": "0.23.9-pre", "plan_cache_hits": 296 } }, { "role": "printed", "runtime": "onnx-w4-block16", "source_checkpoint_sha256": "28c4ca8f9fbbc8c578bb130d997598d2e9c2c4a2dad0b22cb5cc06818ecb988a", "model_sha256": "7f4d08254381c298ce8b88966c4f5b905c7a57baa6f4f8aaf170986874dfc477", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.987012987012987, "crops_per_second": 5.109953854678509, "buckets": { "english": { "cases": 50, "character_parity": 0.9986076301865776, "exact_line_parity": 0.9, "candidate_CER": 0.001392757660167131, "source_CER": 0.0011142061281337048, "CER_delta": 0.0002785515320334262 }, "handwriting": { "cases": 50, "character_parity": 0.9958911145351823, "exact_line_parity": 0.88, "candidate_CER": 0.011253196930946292, "source_CER": 0.011253196930946292, "CER_delta": 0.0 }, "mixed": { "cases": 50, "character_parity": 0.9958096378329842, "exact_line_parity": 0.92, "candidate_CER": 0.010698365527488856, "source_CER": 0.008023774145616641, "CER_delta": 0.002674591381872214 }, "newspaper": { "cases": 50, "character_parity": 0.9986078886310905, "exact_line_parity": 0.96, "candidate_CER": 0.0287569573283859, "source_CER": 0.02829313543599258, "CER_delta": 0.00046382189239332097 }, "printed": { "cases": 50, "character_parity": 0.9970760233918129, "exact_line_parity": 0.92, "candidate_CER": 0.0058309037900874635, "source_CER": 0.004373177842565598, "CER_delta": 0.0014577259475218659 }, "synthetic": { "cases": 50, "character_parity": 0.987012987012987, "exact_line_parity": 0.8, "candidate_CER": 0.05496707701116519, "source_CER": 0.0566847981677641, "CER_delta": -0.0017177211565989122 } }, "execution": { "runtime_version": "1.31.0", "platform": "macOS-26.5.1-arm64-arm-64bit", "threads": 2, "batch": 1, "timing_scope": "Preprocessed crop -> compact tokens/confidence; excludes model load, preprocessing, CTC/bidi decode; caches disabled; first shape warmed only", "provider": "CPUExecutionProvider", "actual_profile_providers": { "CPUExecutionProvider": 370 }, "baseline_sha256": "ae72ecf55cff6414f7a006b15cbbd310f6ef021d1a6adb2fb897660b29332b25", "manifest_sha256": "7e7049a3b56323588ae1f6bd38b1753afc2b1149d275c6f0f07c6993be991978" } }, { "role": "printed", "runtime": "litert-dynamic4-block32", "source_checkpoint_sha256": "28c4ca8f9fbbc8c578bb130d997598d2e9c2c4a2dad0b22cb5cc06818ecb988a", "model_sha256": "89275286e73e2f2bd299caf0a9d4323c8ec00a4cb6817ad1ff132dc9219456f9", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9825855962219598, "crops_per_second": 42.83810617735317, "buckets": { "english": { "cases": 50, "character_parity": 0.9980506822612085, "exact_line_parity": 0.88, "candidate_CER": 0.0019498607242339832, "source_CER": 0.0011142061281337048, "CER_delta": 0.0008356545961002785 }, "handwriting": { "cases": 50, "character_parity": 0.9907550077041603, "exact_line_parity": 0.72, "candidate_CER": 0.01432225063938619, "source_CER": 0.011253196930946292, "CER_delta": 0.003069053708439898 }, "mixed": { "cases": 50, "character_parity": 0.9961089494163424, "exact_line_parity": 0.9, "candidate_CER": 0.00861812778603269, "source_CER": 0.008023774145616641, "CER_delta": 0.0005943536404160475 }, "newspaper": { "cases": 50, "character_parity": 0.997215777262181, "exact_line_parity": 0.92, "candidate_CER": 0.030148423005565864, "source_CER": 0.02829313543599258, "CER_delta": 0.0018552875695732839 }, "printed": { "cases": 50, "character_parity": 0.9941520467836258, "exact_line_parity": 0.86, "candidate_CER": 0.008017492711370262, "source_CER": 0.004373177842565598, "CER_delta": 0.0036443148688046646 }, "synthetic": { "cases": 50, "character_parity": 0.9825855962219598, "exact_line_parity": 0.74, "candidate_CER": 0.05954766676209562, "source_CER": 0.0566847981677641, "CER_delta": 0.0028628685943315204 } }, "execution": { "platform": "Linux-7.0.0-31-generic-x86_64-with-glibc2.43", "threads": 2, "batch": 1, "timing_scope": "Preprocessed crop -> compact tokens/confidence; excludes model load, preprocessing, CTC/bidi decode; caches disabled; first shape warmed only", "provider": "CPUExecutionProvider", "baseline_sha256": "ae72ecf55cff6414f7a006b15cbbd310f6ef021d1a6adb2fb897660b29332b25", "manifest_sha256": "7e7049a3b56323588ae1f6bd38b1753afc2b1149d275c6f0f07c6993be991978" } }, { "role": "printed", "runtime": "tract-metal-w4-block16", "source_checkpoint_sha256": "28c4ca8f9fbbc8c578bb130d997598d2e9c2c4a2dad0b22cb5cc06818ecb988a", "model_sha256": "7f4d08254381c298ce8b88966c4f5b905c7a57baa6f4f8aaf170986874dfc477", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9867178276269185, "crops_per_second": 14.093633328198088, "buckets": { "english": { "cases": 50, "character_parity": 0.9986076301865776, "exact_line_parity": 0.9, "candidate_CER": 0.001392757660167131, "source_CER": 0.0011142061281337048, "CER_delta": 0.0002785515320334262 }, "handwriting": { "cases": 50, "character_parity": 0.9953775038520801, "exact_line_parity": 0.86, "candidate_CER": 0.011764705882352941, "source_CER": 0.011253196930946292, "CER_delta": 0.0005115089514066496 }, "mixed": { "cases": 50, "character_parity": 0.9958096378329842, "exact_line_parity": 0.92, "candidate_CER": 0.010698365527488856, "source_CER": 0.008023774145616641, "CER_delta": 0.002674591381872214 }, "newspaper": { "cases": 50, "character_parity": 0.9986078886310905, "exact_line_parity": 0.96, "candidate_CER": 0.0287569573283859, "source_CER": 0.02829313543599258, "CER_delta": 0.00046382189239332097 }, "printed": { "cases": 50, "character_parity": 0.9970760233918129, "exact_line_parity": 0.92, "candidate_CER": 0.0058309037900874635, "source_CER": 0.004373177842565598, "CER_delta": 0.0014577259475218659 }, "synthetic": { "cases": 50, "character_parity": 0.9867178276269185, "exact_line_parity": 0.8, "candidate_CER": 0.05496707701116519, "source_CER": 0.0566847981677641, "CER_delta": -0.0017177211565989122 } }, "execution": { "timing_scope": "Batch1, native model time after plan warmup; excludes plan preparation, input IO, CTC/bidi; exact-result cache off", "version": "0.23.9-pre", "plan_cache_hits": 296 } }, { "role": "degraded", "runtime": "browser-wasm", "source_checkpoint_sha256": "8867b309e881b698685e5b6e06a37c9f495e04dcf7027a044ad44c13cf8abd64", "model_sha256": "f9feca4a76053afea6640e46f66cb3b4709018ca3fe1a599a9e52524c6e24892", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9879269729093051, "crops_per_second": 0.4708207395006966, "buckets": { "english": { "cases": 50, "character_parity": 0.9994432071269488, "exact_line_parity": 0.96, "candidate_CER": 0.0008356545961002785, "source_CER": 0.0008356545961002785, "CER_delta": 0.0 }, "handwriting": { "cases": 50, "character_parity": 0.9943502824858758, "exact_line_parity": 0.82, "candidate_CER": 0.010741687979539642, "source_CER": 0.010741687979539642, "CER_delta": 0.0 }, "mixed": { "cases": 50, "character_parity": 0.9952110146662676, "exact_line_parity": 0.92, "candidate_CER": 0.00950965824665676, "source_CER": 0.008320950965824666, "CER_delta": 0.001188707280832095 }, "newspaper": { "cases": 50, "character_parity": 0.9995366079703429, "exact_line_parity": 0.98, "candidate_CER": 0.02736549165120594, "source_CER": 0.027829313543599257, "CER_delta": -0.00046382189239332097 }, "printed": { "cases": 50, "character_parity": 0.997808619430241, "exact_line_parity": 0.94, "candidate_CER": 0.00510204081632653, "source_CER": 0.00510204081632653, "CER_delta": 0.0 }, "synthetic": { "cases": 50, "character_parity": 0.9879269729093051, "exact_line_parity": 0.78, "candidate_CER": 0.05582593758946464, "source_CER": 0.05697108502719725, "CER_delta": -0.001145147437732608 } }, "execution": { "threads": 1, "timing_scope": "Warm bucket, preprocessed crop to compact outputs, exact result cache disabled; concurrent Mac validation jobs may contend" } }, { "role": "degraded", "runtime": "onnx-linux-cpu-hqq4-block32", "source_checkpoint_sha256": "8867b309e881b698685e5b6e06a37c9f495e04dcf7027a044ad44c13cf8abd64", "model_sha256": "5094b4720adf8436833e862eec1a90eadff60215563bce2dbb2f6a43cee17020", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9870435806831567, "crops_per_second": 13.465028526823433, "buckets": { "english": { "cases": 50, "character_parity": 0.9991648106904232, "exact_line_parity": 0.94, "candidate_CER": 0.0011142061281337048, "source_CER": 0.0008356545961002785, "CER_delta": 0.0002785515320334262 }, "handwriting": { "cases": 50, "character_parity": 0.9948638931689779, "exact_line_parity": 0.88, "candidate_CER": 0.010230179028132993, "source_CER": 0.010741687979539642, "CER_delta": -0.0005115089514066496 }, "mixed": { "cases": 50, "character_parity": 0.9958096378329842, "exact_line_parity": 0.92, "candidate_CER": 0.01099554234769688, "source_CER": 0.008320950965824666, "CER_delta": 0.002674591381872214 }, "newspaper": { "cases": 50, "character_parity": 0.9986098239110287, "exact_line_parity": 0.94, "candidate_CER": 0.02829313543599258, "source_CER": 0.027829313543599257, "CER_delta": 0.00046382189239332097 }, "printed": { "cases": 50, "character_parity": 0.9992695398100804, "exact_line_parity": 0.98, "candidate_CER": 0.0058309037900874635, "source_CER": 0.00510204081632653, "CER_delta": 0.0007288629737609329 }, "synthetic": { "cases": 50, "character_parity": 0.9870435806831567, "exact_line_parity": 0.8, "candidate_CER": 0.05811623246492986, "source_CER": 0.05697108502719725, "CER_delta": 0.001145147437732608 } }, "execution": { "runtime_version": "1.31.0", "platform": "Linux-7.0.0-31-generic-x86_64-with-glibc2.43", "threads": 2, "batch": 1, "timing_scope": "Preprocessed crop -> compact tokens/confidence; excludes model load, preprocessing, CTC/bidi decode; caches disabled; first shape warmed only", "provider": "CPUExecutionProvider", "actual_profile_providers": { "CPUExecutionProvider": 371 }, "baseline_sha256": "27c22f9b43892bd4275d414c22d06ec0507a1fe3e365fed97471069bfcbafca1", "manifest_sha256": "7e7049a3b56323588ae1f6bd38b1753afc2b1149d275c6f0f07c6993be991978" } }, { "role": "degraded", "runtime": "coreml-w4-block16", "source_checkpoint_sha256": "8867b309e881b698685e5b6e06a37c9f495e04dcf7027a044ad44c13cf8abd64", "model_sha256": null, "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9861601884570083, "crops_per_second": 78.93670187737018, "buckets": { "english": { "cases": 50, "character_parity": 0.9994432071269488, "exact_line_parity": 0.96, "candidate_CER": 0.001392757660167131, "source_CER": 0.0008356545961002785, "CER_delta": 0.0005571030640668524 }, "handwriting": { "cases": 50, "character_parity": 0.9953775038520801, "exact_line_parity": 0.84, "candidate_CER": 0.01329923273657289, "source_CER": 0.010741687979539642, "CER_delta": 0.0025575447570332483 }, "mixed": { "cases": 50, "character_parity": 0.996707572583059, "exact_line_parity": 0.9, "candidate_CER": 0.007429420505200594, "source_CER": 0.008320950965824666, "CER_delta": -0.0008915304606240713 }, "newspaper": { "cases": 50, "character_parity": 0.9990732159406858, "exact_line_parity": 0.96, "candidate_CER": 0.027829313543599257, "source_CER": 0.027829313543599257, "CER_delta": 0.0 }, "printed": { "cases": 50, "character_parity": 0.9963476990504018, "exact_line_parity": 0.92, "candidate_CER": 0.006559766763848397, "source_CER": 0.00510204081632653, "CER_delta": 0.0014577259475218659 }, "synthetic": { "cases": 50, "character_parity": 0.9861601884570083, "exact_line_parity": 0.74, "candidate_CER": 0.0572573718866304, "source_CER": 0.05697108502719725, "CER_delta": 0.000286286859433152 } }, "execution": { "platform": "macOS-26.5.1-arm64-arm-64bit", "threads": 2, "batch": 1, "timing_scope": "Preprocessed crop -> compact tokens/confidence; excludes model load, preprocessing, CTC/bidi decode; caches disabled; first shape warmed only", "provider": "CPUExecutionProvider", "baseline_sha256": "27c22f9b43892bd4275d414c22d06ec0507a1fe3e365fed97471069bfcbafca1", "manifest_sha256": "7e7049a3b56323588ae1f6bd38b1753afc2b1149d275c6f0f07c6993be991978" } }, { "role": "degraded", "runtime": "onnx-hqq4-block32", "source_checkpoint_sha256": "8867b309e881b698685e5b6e06a37c9f495e04dcf7027a044ad44c13cf8abd64", "model_sha256": "5094b4720adf8436833e862eec1a90eadff60215563bce2dbb2f6a43cee17020", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9870435806831567, "crops_per_second": 9.594292903740188, "buckets": { "english": { "cases": 50, "character_parity": 0.9991648106904232, "exact_line_parity": 0.94, "candidate_CER": 0.0011142061281337048, "source_CER": 0.0008356545961002785, "CER_delta": 0.0002785515320334262 }, "handwriting": { "cases": 50, "character_parity": 0.9948638931689779, "exact_line_parity": 0.88, "candidate_CER": 0.010230179028132993, "source_CER": 0.010741687979539642, "CER_delta": -0.0005115089514066496 }, "mixed": { "cases": 50, "character_parity": 0.9958096378329842, "exact_line_parity": 0.92, "candidate_CER": 0.01099554234769688, "source_CER": 0.008320950965824666, "CER_delta": 0.002674591381872214 }, "newspaper": { "cases": 50, "character_parity": 0.9986098239110287, "exact_line_parity": 0.94, "candidate_CER": 0.02829313543599258, "source_CER": 0.027829313543599257, "CER_delta": 0.00046382189239332097 }, "printed": { "cases": 50, "character_parity": 0.9992695398100804, "exact_line_parity": 0.98, "candidate_CER": 0.0058309037900874635, "source_CER": 0.00510204081632653, "CER_delta": 0.0007288629737609329 }, "synthetic": { "cases": 50, "character_parity": 0.9870435806831567, "exact_line_parity": 0.8, "candidate_CER": 0.05811623246492986, "source_CER": 0.05697108502719725, "CER_delta": 0.001145147437732608 } }, "execution": { "runtime_version": "1.31.0", "platform": "macOS-26.5.1-arm64-arm-64bit", "threads": 2, "batch": 1, "timing_scope": "Preprocessed crop -> compact tokens/confidence; excludes model load, preprocessing, CTC/bidi decode; caches disabled; first shape warmed only", "provider": "CPUExecutionProvider", "actual_profile_providers": { "CPUExecutionProvider": 370 }, "baseline_sha256": "27c22f9b43892bd4275d414c22d06ec0507a1fe3e365fed97471069bfcbafca1", "manifest_sha256": "7e7049a3b56323588ae1f6bd38b1753afc2b1149d275c6f0f07c6993be991978" } }, { "role": "degraded", "runtime": "tract-linux-cuda-w4-block16", "source_checkpoint_sha256": "8867b309e881b698685e5b6e06a37c9f495e04dcf7027a044ad44c13cf8abd64", "model_sha256": "f9feca4a76053afea6640e46f66cb3b4709018ca3fe1a599a9e52524c6e24892", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9879269729093051, "crops_per_second": 216.53247316857357, "buckets": { "english": { "cases": 50, "character_parity": 0.9994432071269488, "exact_line_parity": 0.96, "candidate_CER": 0.0008356545961002785, "source_CER": 0.0008356545961002785, "CER_delta": 0.0 }, "handwriting": { "cases": 50, "character_parity": 0.9943502824858758, "exact_line_parity": 0.82, "candidate_CER": 0.010741687979539642, "source_CER": 0.010741687979539642, "CER_delta": 0.0 }, "mixed": { "cases": 50, "character_parity": 0.9952110146662676, "exact_line_parity": 0.92, "candidate_CER": 0.00950965824665676, "source_CER": 0.008320950965824666, "CER_delta": 0.001188707280832095 }, "newspaper": { "cases": 50, "character_parity": 0.9995366079703429, "exact_line_parity": 0.98, "candidate_CER": 0.02736549165120594, "source_CER": 0.027829313543599257, "CER_delta": -0.00046382189239332097 }, "printed": { "cases": 50, "character_parity": 0.997808619430241, "exact_line_parity": 0.94, "candidate_CER": 0.00510204081632653, "source_CER": 0.00510204081632653, "CER_delta": 0.0 }, "synthetic": { "cases": 50, "character_parity": 0.9879269729093051, "exact_line_parity": 0.78, "candidate_CER": 0.05582593758946464, "source_CER": 0.05697108502719725, "CER_delta": -0.001145147437732608 } }, "execution": { "timing_scope": "Batch1, native model time after plan warmup; excludes plan preparation, input IO, CTC/bidi; exact-result cache off", "version": "0.23.9-pre", "plan_cache_hits": 296 } }, { "role": "degraded", "runtime": "browser-webgpu", "source_checkpoint_sha256": "8867b309e881b698685e5b6e06a37c9f495e04dcf7027a044ad44c13cf8abd64", "model_sha256": "f9feca4a76053afea6640e46f66cb3b4709018ca3fe1a599a9e52524c6e24892", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9879269729093051, "crops_per_second": 5.805356834741982, "buckets": { "english": { "cases": 50, "character_parity": 0.9994432071269488, "exact_line_parity": 0.96, "candidate_CER": 0.0008356545961002785, "source_CER": 0.0008356545961002785, "CER_delta": 0.0 }, "handwriting": { "cases": 50, "character_parity": 0.9943502824858758, "exact_line_parity": 0.82, "candidate_CER": 0.010741687979539642, "source_CER": 0.010741687979539642, "CER_delta": 0.0 }, "mixed": { "cases": 50, "character_parity": 0.9952110146662676, "exact_line_parity": 0.92, "candidate_CER": 0.00950965824665676, "source_CER": 0.008320950965824666, "CER_delta": 0.001188707280832095 }, "newspaper": { "cases": 50, "character_parity": 0.9995366079703429, "exact_line_parity": 0.98, "candidate_CER": 0.02736549165120594, "source_CER": 0.027829313543599257, "CER_delta": -0.00046382189239332097 }, "printed": { "cases": 50, "character_parity": 0.997808619430241, "exact_line_parity": 0.94, "candidate_CER": 0.00510204081632653, "source_CER": 0.00510204081632653, "CER_delta": 0.0 }, "synthetic": { "cases": 50, "character_parity": 0.9879269729093051, "exact_line_parity": 0.78, "candidate_CER": 0.05582593758946464, "source_CER": 0.05697108502719725, "CER_delta": -0.001145147437732608 } }, "execution": { "threads": 2, "timing_scope": "Warm bucket, preprocessed crop to compact outputs, exact result cache disabled; concurrent Mac validation jobs may contend" } }, { "role": "degraded", "runtime": "onnx-linux-cuda-w4-block16", "source_checkpoint_sha256": "8867b309e881b698685e5b6e06a37c9f495e04dcf7027a044ad44c13cf8abd64", "model_sha256": "f9feca4a76053afea6640e46f66cb3b4709018ca3fe1a599a9e52524c6e24892", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9879269729093051, "crops_per_second": 323.7218798515477, "buckets": { "english": { "cases": 50, "character_parity": 0.9994432071269488, "exact_line_parity": 0.96, "candidate_CER": 0.0008356545961002785, "source_CER": 0.0008356545961002785, "CER_delta": 0.0 }, "handwriting": { "cases": 50, "character_parity": 0.9943502824858758, "exact_line_parity": 0.82, "candidate_CER": 0.010741687979539642, "source_CER": 0.010741687979539642, "CER_delta": 0.0 }, "mixed": { "cases": 50, "character_parity": 0.9952110146662676, "exact_line_parity": 0.92, "candidate_CER": 0.00950965824665676, "source_CER": 0.008320950965824666, "CER_delta": 0.001188707280832095 }, "newspaper": { "cases": 50, "character_parity": 0.9995366079703429, "exact_line_parity": 0.98, "candidate_CER": 0.02736549165120594, "source_CER": 0.027829313543599257, "CER_delta": -0.00046382189239332097 }, "printed": { "cases": 50, "character_parity": 0.997808619430241, "exact_line_parity": 0.94, "candidate_CER": 0.00510204081632653, "source_CER": 0.00510204081632653, "CER_delta": 0.0 }, "synthetic": { "cases": 50, "character_parity": 0.9879269729093051, "exact_line_parity": 0.78, "candidate_CER": 0.05582593758946464, "source_CER": 0.05697108502719725, "CER_delta": -0.001145147437732608 } }, "execution": { "runtime_version": "1.23.2", "platform": "Linux-7.0.0-31-generic-x86_64-with-glibc2.43", "threads": 2, "batch": 1, "timing_scope": "Preprocessed crop -> compact tokens/confidence; excludes model load, preprocessing, CTC/bidi decode; caches disabled; first shape warmed only", "provider": "CUDAExecutionProvider", "actual_profile_providers": { "CUDAExecutionProvider": 448, "CPUExecutionProvider": 3 }, "baseline_sha256": "27c22f9b43892bd4275d414c22d06ec0507a1fe3e365fed97471069bfcbafca1", "manifest_sha256": "7e7049a3b56323588ae1f6bd38b1753afc2b1149d275c6f0f07c6993be991978" } }, { "role": "degraded", "runtime": "tract-cpu-w4-block16", "source_checkpoint_sha256": "8867b309e881b698685e5b6e06a37c9f495e04dcf7027a044ad44c13cf8abd64", "model_sha256": "f9feca4a76053afea6640e46f66cb3b4709018ca3fe1a599a9e52524c6e24892", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9879269729093051, "crops_per_second": 27.57683044201156, "buckets": { "english": { "cases": 50, "character_parity": 0.9994432071269488, "exact_line_parity": 0.96, "candidate_CER": 0.0008356545961002785, "source_CER": 0.0008356545961002785, "CER_delta": 0.0 }, "handwriting": { "cases": 50, "character_parity": 0.9943502824858758, "exact_line_parity": 0.82, "candidate_CER": 0.010741687979539642, "source_CER": 0.010741687979539642, "CER_delta": 0.0 }, "mixed": { "cases": 50, "character_parity": 0.9952110146662676, "exact_line_parity": 0.92, "candidate_CER": 0.00950965824665676, "source_CER": 0.008320950965824666, "CER_delta": 0.001188707280832095 }, "newspaper": { "cases": 50, "character_parity": 0.9995366079703429, "exact_line_parity": 0.98, "candidate_CER": 0.02736549165120594, "source_CER": 0.027829313543599257, "CER_delta": -0.00046382189239332097 }, "printed": { "cases": 50, "character_parity": 0.997808619430241, "exact_line_parity": 0.94, "candidate_CER": 0.00510204081632653, "source_CER": 0.00510204081632653, "CER_delta": 0.0 }, "synthetic": { "cases": 50, "character_parity": 0.9879269729093051, "exact_line_parity": 0.78, "candidate_CER": 0.05582593758946464, "source_CER": 0.05697108502719725, "CER_delta": -0.001145147437732608 } }, "execution": { "timing_scope": "Batch1, native model time after plan warmup; excludes plan preparation, input IO, CTC/bidi; exact-result cache off", "version": "0.23.9-pre", "plan_cache_hits": 296 } }, { "role": "degraded", "runtime": "onnx-w4-block16", "source_checkpoint_sha256": "8867b309e881b698685e5b6e06a37c9f495e04dcf7027a044ad44c13cf8abd64", "model_sha256": "f9feca4a76053afea6640e46f66cb3b4709018ca3fe1a599a9e52524c6e24892", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9885159010600707, "crops_per_second": 7.316526044854179, "buckets": { "english": { "cases": 50, "character_parity": 0.9994432071269488, "exact_line_parity": 0.96, "candidate_CER": 0.0008356545961002785, "source_CER": 0.0008356545961002785, "CER_delta": 0.0 }, "handwriting": { "cases": 50, "character_parity": 0.9943502824858758, "exact_line_parity": 0.82, "candidate_CER": 0.010741687979539642, "source_CER": 0.010741687979539642, "CER_delta": 0.0 }, "mixed": { "cases": 50, "character_parity": 0.9952110146662676, "exact_line_parity": 0.92, "candidate_CER": 0.009212481426448737, "source_CER": 0.008320950965824666, "CER_delta": 0.0008915304606240713 }, "newspaper": { "cases": 50, "character_parity": 0.9995366079703429, "exact_line_parity": 0.98, "candidate_CER": 0.02736549165120594, "source_CER": 0.027829313543599257, "CER_delta": -0.00046382189239332097 }, "printed": { "cases": 50, "character_parity": 0.997808619430241, "exact_line_parity": 0.94, "candidate_CER": 0.00510204081632653, "source_CER": 0.00510204081632653, "CER_delta": 0.0 }, "synthetic": { "cases": 50, "character_parity": 0.9885159010600707, "exact_line_parity": 0.78, "candidate_CER": 0.055253363870598336, "source_CER": 0.05697108502719725, "CER_delta": -0.0017177211565989122 } }, "execution": { "runtime_version": "1.31.0", "platform": "macOS-26.5.1-arm64-arm-64bit", "threads": 2, "batch": 1, "timing_scope": "Preprocessed crop -> compact tokens/confidence; excludes model load, preprocessing, CTC/bidi decode; caches disabled; first shape warmed only", "provider": "CPUExecutionProvider", "actual_profile_providers": { "CPUExecutionProvider": 370 }, "baseline_sha256": "27c22f9b43892bd4275d414c22d06ec0507a1fe3e365fed97471069bfcbafca1", "manifest_sha256": "7e7049a3b56323588ae1f6bd38b1753afc2b1149d275c6f0f07c6993be991978" } }, { "role": "degraded", "runtime": "litert-dynamic4-block32", "source_checkpoint_sha256": "8867b309e881b698685e5b6e06a37c9f495e04dcf7027a044ad44c13cf8abd64", "model_sha256": "8201f9eb76f82ec573b7109bf19a3b02718b11d89891adb76c4dd09cff4fa0bb", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9826266195524146, "crops_per_second": 43.29946678010443, "buckets": { "english": { "cases": 50, "character_parity": 0.9988864142538976, "exact_line_parity": 0.94, "candidate_CER": 0.001392757660167131, "source_CER": 0.0008356545961002785, "CER_delta": 0.0005571030640668524 }, "handwriting": { "cases": 50, "character_parity": 0.9912686183872624, "exact_line_parity": 0.72, "candidate_CER": 0.014833759590792838, "source_CER": 0.010741687979539642, "CER_delta": 0.004092071611253197 }, "mixed": { "cases": 50, "character_parity": 0.9961089494163424, "exact_line_parity": 0.88, "candidate_CER": 0.009212481426448737, "source_CER": 0.008320950965824666, "CER_delta": 0.0008915304606240713 }, "newspaper": { "cases": 50, "character_parity": 0.995366079703429, "exact_line_parity": 0.92, "candidate_CER": 0.03153988868274583, "source_CER": 0.027829313543599257, "CER_delta": 0.0037105751391465678 }, "printed": { "cases": 50, "character_parity": 0.9948867786705624, "exact_line_parity": 0.88, "candidate_CER": 0.008746355685131196, "source_CER": 0.00510204081632653, "CER_delta": 0.0036443148688046646 }, "synthetic": { "cases": 50, "character_parity": 0.9826266195524146, "exact_line_parity": 0.64, "candidate_CER": 0.06212424849699399, "source_CER": 0.05697108502719725, "CER_delta": 0.005153163469796736 } }, "execution": { "platform": "Linux-7.0.0-31-generic-x86_64-with-glibc2.43", "threads": 2, "batch": 1, "timing_scope": "Preprocessed crop -> compact tokens/confidence; excludes model load, preprocessing, CTC/bidi decode; caches disabled; first shape warmed only", "provider": "CPUExecutionProvider", "baseline_sha256": "27c22f9b43892bd4275d414c22d06ec0507a1fe3e365fed97471069bfcbafca1", "manifest_sha256": "7e7049a3b56323588ae1f6bd38b1753afc2b1149d275c6f0f07c6993be991978" } }, { "role": "degraded", "runtime": "tract-metal-w4-block16", "source_checkpoint_sha256": "8867b309e881b698685e5b6e06a37c9f495e04dcf7027a044ad44c13cf8abd64", "model_sha256": "f9feca4a76053afea6640e46f66cb3b4709018ca3fe1a599a9e52524c6e24892", "cases": 300, "parity_gate": true, "worst_bucket_character_parity": 0.9879269729093051, "crops_per_second": 43.570941984219765, "buckets": { "english": { "cases": 50, "character_parity": 0.9994432071269488, "exact_line_parity": 0.96, "candidate_CER": 0.0008356545961002785, "source_CER": 0.0008356545961002785, "CER_delta": 0.0 }, "handwriting": { "cases": 50, "character_parity": 0.9943502824858758, "exact_line_parity": 0.82, "candidate_CER": 0.010741687979539642, "source_CER": 0.010741687979539642, "CER_delta": 0.0 }, "mixed": { "cases": 50, "character_parity": 0.9952110146662676, "exact_line_parity": 0.92, "candidate_CER": 0.00950965824665676, "source_CER": 0.008320950965824666, "CER_delta": 0.001188707280832095 }, "newspaper": { "cases": 50, "character_parity": 0.9995366079703429, "exact_line_parity": 0.98, "candidate_CER": 0.02736549165120594, "source_CER": 0.027829313543599257, "CER_delta": -0.00046382189239332097 }, "printed": { "cases": 50, "character_parity": 0.997808619430241, "exact_line_parity": 0.94, "candidate_CER": 0.00510204081632653, "source_CER": 0.00510204081632653, "CER_delta": 0.0 }, "synthetic": { "cases": 50, "character_parity": 0.9879269729093051, "exact_line_parity": 0.78, "candidate_CER": 0.05582593758946464, "source_CER": 0.05697108502719725, "CER_delta": -0.001145147437732608 } }, "execution": { "timing_scope": "Batch1, native model time after plan warmup; excludes plan preparation, input IO, CTC/bidi; exact-result cache off", "version": "0.23.9-pre", "plan_cache_hits": 296 } } ], "detector_candidates": [ { "role": "main_detector", "runtime": "onnx-detector-w4-allconv", "mean_text_mask_iou": 0.965475773488656, "min_text_mask_iou": 0.9336849044166118, "transcription_parity_qualified": false }, { "role": "main_detector", "runtime": "litert-detector-weight-only4", "mean_text_mask_iou": 0.9595448080555493, "min_text_mask_iou": 0.9307568438003221, "transcription_parity_qualified": false }, { "role": "main_detector", "runtime": "onnx-detector-w4-pointwise", "mean_text_mask_iou": 0.9773309403974731, "min_text_mask_iou": 0.9627226287449951, "transcription_parity_qualified": false }, { "role": "main_detector", "runtime": "coreml-detector-w4-block16", "mean_text_mask_iou": 0.9810492286988853, "min_text_mask_iou": 0.9689657529564564, "transcription_parity_qualified": false }, { "role": "line_guard", "runtime": "onnx-detector-w4-allconv", "mean_text_mask_iou": 0.9665194888564727, "min_text_mask_iou": 0.9380598925006399, "transcription_parity_qualified": false }, { "role": "line_guard", "runtime": "litert-detector-weight-only4", "mean_text_mask_iou": 0.9555789912494387, "min_text_mask_iou": 0.9315724097618003, "transcription_parity_qualified": false }, { "role": "line_guard", "runtime": "onnx-detector-w4-pointwise", "mean_text_mask_iou": 0.981330133565763, "min_text_mask_iou": 0.9714587737843552, "transcription_parity_qualified": false }, { "role": "line_guard", "runtime": "coreml-detector-w4-block16", "mean_text_mask_iou": 0.9766960486901575, "min_text_mask_iou": 0.9615810418599084, "transcription_parity_qualified": false }, { "role": "printed_detector", "runtime": "onnx-detector-w4-allconv", "mean_text_mask_iou": 0.9617834259340746, "min_text_mask_iou": 0.9151712887438825, "transcription_parity_qualified": false }, { "role": "printed_detector", "runtime": "litert-detector-weight-only4", "mean_text_mask_iou": 0.9542730677661033, "min_text_mask_iou": 0.9115772488906817, "transcription_parity_qualified": false }, { "role": "printed_detector", "runtime": "onnx-detector-w4-pointwise", "mean_text_mask_iou": 0.9855684873828712, "min_text_mask_iou": 0.9750303684707787, "transcription_parity_qualified": false }, { "role": "printed_detector", "runtime": "coreml-detector-w4-block16", "mean_text_mask_iou": 0.9764452196609898, "min_text_mask_iou": 0.9485420240137221, "transcription_parity_qualified": false } ], "public_page_diagnostics": { "page-quant-cpu.json": { "provider": "CPUExecutionProvider", "load_seconds": 4.683880491997115, "seconds": 19.141634956933558, "character_parity": 1.0, "parity_gate": true, "source_characters": 140, "candidate": { "text": "نمونه آزمایشی بینا ۰٫۴ ریزه\n\nاین صفحه برای آزمودن نصب ساخته شده است.\n\nشماره: ۲۷۴\nتاریخ: ۱۴۰۵/۰۸/۱۶\n\nخواندن متن فارسی چاپی، عددها و نشانه‌ها.", "lines": [ { "text": "نمونه آزمایشی بینا ۰٫۴ ریزه", "layer": null, "box": [ 699.0, 79.0, 1164.0, 130.0 ], "confidence": 0.9928557872772217 }, { "text": "این صفحه برای آزمودن نصب ساخته شده است.", "layer": null, "box": [ 536.5, 187.5, 1163.0, 228.0 ], "confidence": 0.9855734705924988 }, { "text": "شماره: ۲۷۴", "layer": null, "box": [ 720.0, 300.0, 882.5, 337.0 ], "confidence": 1.0 }, { "text": "تاریخ: ۱۴۰۵/۰۸/۱۶", "layer": null, "box": [ 917.0, 298.5, 1163.5, 339.0 ], "confidence": 0.9910417199134827 }, { "text": "خواندن متن فارسی چاپی، عددها و نشانه‌ها.", "layer": null, "box": [ 592.0, 407.5, 1163.5, 450.0 ], "confidence": 0.9801881909370422 } ] }, "repeat_text_identical": true, "reader_calls_on_repeat": 0, "readers": { "body": { "model_calls": 1, "cache_hits": 5 }, "short_labels": { "model_calls": 0, "cache_hits": 0 }, "printed": { "model_calls": 1, "cache_hits": 5 }, "numeric_math": { "model_calls": 0, "cache_hits": 0 }, "degraded": { "model_calls": 0, "cache_hits": 0 } }, "scope": "One public whole-page demo; not the full protected page/table gate" }, "public-pages-quant.json": { "rows": [ { "image": "printed-modern.png", "text": "بینا: خواندن متن فارسی بدون اینترنت\n\nبینا برنامه‌ای است که متن فارسی را از عکس و فایل PDF می‌خواند و به متن قابل\nویرایش تبدیل می‌کند. همۀ کار روی همین رایانه انجام می‌شود و هیچ تصویری به جای\nدیگری فرستاده نمی‌شود؛ برای همین می‌توان نامه‌های اداری، پرونده‌های شخصی و\nاسناد محرمانه را بی‌نگرانی به آن سپرد.\n\nمدل هوش مصنوعی بینا با هزاران صفحۀ چاپی و دست‌نویس آموزش دیده است. این\nمدل صفحه را به چند بخش تقسیم می‌کند، هر بخش را جداگانه می‌خواند و در پایان\nمتن کامل را به همان ترتیب خواندن کنار هم می‌گذارد.\n\nاگر رایانه کارت گرافیک انویدیا داشته باشد، بینا می‌تواند از آن کمک بگیرد و هر صفحه را\nچند برابر سریع‌تر بخواند. در غیر این صورت پردازندۀ مرکزی همین کار را با همان دقت\nانجام می‌دهد.", "lines": 11, "seconds": 19.679442223976366, "table_grids": 0, "table_errors": [], "character_parity": 1.0, "parity_gate": true }, { "image": "hand-plain.jpg", "text": "یک سال بعد از این کار دلیل این امر هم گستردگی\nمحصولات این شرکت بود این حافظه می‌تواند توسط نرم‌افزارهای\nارتباط مورد استفاده قرار بگیرند در این مجموعه دستوری از\nقواعد دستوری زیر استفاده می‌شود یک رشته می‌تواند شامل چندین\nدستور های س، باشد\nنرم‌افزار کاربردی نام عمومی برنامه‌های کامپیوتری برای\nانجام وظایف است نرم‌افزار منبع باز نرم‌افزاری، است\nکه کد منبع آن تحت یک مجوز خاص در اختیار\nدارندگان مجوز آن قرار می‌گیرد می‌اوان از آن برای\nهر هدفی استفاده و، منتشر کرد کد منبع باز\nاست و، می‌اوان آن را در صورت نیاز تغییر داد اگر\nهر دو نرم‌افزار از یک مجوز یکسان استفاده کنند معمولا\nقابل ادغام هستند مراقبت‌های بهداشتی و، کمک‌های\nاجتماعی برنامه‌های داخلی و، اختصاصی خط تجاری\nفعالیت این شرکت در راستای تولید نرم‌افزار و،\nبرنامه‌های کاربردی برای موبایل بود که در\nاین راه موفقیت بسیاری کسب کرده بود این سیستم عامل\nدر ابتدا ساختار بسیار ساده‌ای داشت و، از برنامه‌های\nمحدودی پشتیبانی می‌کرد اولین نسخه این سیستم عامل در", "lines": 19, "seconds": 19.4021920269588, "table_grids": 0, "table_errors": [], "character_parity": 0.9977900552486187, "parity_gate": true }, { "image": "hand-table.jpg", "text": "دبستان .....\nجمهوری اس لامی ایران\nاداره کل آموزش وپرورش استان ......\nمدیریت آموزش وپرورش .....\nفروردین\nوزارت اموزش وپرورش\nاولیا\nامضا آموزگار:\nجنگ جهانی یکم تا اواری دهه ۱۹۳۰ خدمات هوایی\nنیردی درمس و، خدهات هوایی نیروی دریایی یونان\nبا هم یکی شده و، نیروی دریایی یویان را تشکیل\nدارند جذگ جیانی ردوم و، جنگ داخلی نیروی\nهوایی یونان میز نظر وزارت دفاع ولی یونان و،\nمستقیما زر ظر ستاد کل دفاع ملی یونان قرار\nدااد\nقبل از جدک جهانی دوم استفاده بعد از\nجنگ ههانی دوم توکدات پ زاز سنگ\nجهانی ادوم استفاده گسترده توجه سروهای انتظامی و،\nنظمی کهی از جنگت نیریک ژمنی ور چلیس\nاستفاده توسط نیروی زمیمی تا رشه ۱۹۶۰\nاستفاده پس از جنگ جهانی دوم شرانفت شده\nاز ایالات متحده\nنام ونام خانوادگی دانش آموز:\nسال تحصیلی ۰۳-۰۴\nماه\nپایه پنجم\nقرآن\nاسفند\nبهمن\nورزش\nخوب\nخوب\nفصل\nمهر\nضعف\nضکیف\nمتوسط\nمهر وامضا مدیر مدرسه", "lines": 38, "seconds": 15.461470424081199, "table_grids": 0, "table_errors": [], "character_parity": 0.7982989064398542, "parity_gate": false } ], "scope": "Three public Bina Desktop samples; no claim of full gate certification", "readers": { "body": { "calls": 14, "hits": 0 }, "short_labels": { "calls": 3, "hits": 0 }, "printed": { "calls": 27, "hits": 0 }, "numeric_math": { "calls": 0, "hits": 0 }, "degraded": { "calls": 0, "hits": 0 } } }, "public-pages-quant-cpu-block16.json": { "rows": [ { "image": "hand-table.jpg", "text": "دبستان .....\nمدیریت آموزش وپرورش .....\nاداره کل آموزش وپرورش استان .....\nجمهوری اس_ لامی ایران\nفروردین\nوزارت اموزش وپرورش\nفصیل\nژ:مسیتان\nاولیا\nامضا آموزگار:\nجنگ جهانی یکم تا اواشی دهه ۱۹۳۰ خدمات هوایع\nنیردی ترممی و، خدهات هوایی نیروی درپایی یونان\nبا قام یکی شده و، نیروی داریایی یویان را تشکیل\nدارند جنگ جهانی روم و، جنگ داخلی نیروی\nهوای یونان ران نظر وزارت دفاع ملی یونان و،\nمستقیما زر ظر ستاد کل دفاع ملی بونان قرار\nدا د\nقبل از جدن جهانی دوم استفماده بعد از\nجنگ ههانی لوم توهدات پ زان هنگ\nجهانی دوم استفاده گیسترده توسه سروهای انتظامی و،\nنظمی کی از جنگ نیروی ژرمیعی دو، چلیس\nاست‌فاده توسط نیروی زمینی تا رشه ۱۹۶۰\nاستفاده پس از جنگ جهانی دوم درافت شده\nاز ایالات متحده\nسال تحصیلی ۰۳-۰۴\nنام ونام خانوادگی دانش آموز:\n۵\nماه\nمهر\nپایه پنجم\nقرآن\nبهمن\nاسفند\nورزش\nخوب\nخالی\nخوب\nفصل\nمهر\nضعف\nموس\nضکیف\nمترسط\nکوبت دوم\nمهر وامضا مدیر مدرسه", "lines": 45, "seconds": 14.24943418300245, "table_grids": 0, "table_errors": [], "character_parity": 0.8651275820170109, "parity_gate": false } ], "cache_identity": "5804c41c085f3903b79ce585", "scope": "Public Bina Desktop samples; no claim of full gate certification", "readers": { "body": { "calls": 9, "hits": 0, "variant": "block16" }, "short_labels": { "calls": 3, "hits": 0, "variant": "block16" }, "printed": { "calls": 19, "hits": 0, "variant": "block16" }, "numeric_math": { "calls": 0, "hits": 0, "variant": "block16" }, "degraded": { "calls": 0, "hits": 0, "variant": "block16" } } }, "public-pages-quant-cpu-block16-safe.json": { "rows": [ { "image": "printed-modern.png", "text": "بینا: خواندن متن فارسی بدون اینترنت\n\nبینا برنامه‌ای است که متن فارسی را از عکس و فایل PDF می‌خواند و به متن قابل\nویرایش تبدیل می‌کند. همۀ کار روی همین رایانه انجام می‌شود و هیچ تصویری به جای\nدیگری فرستاده نمی‌شود؛ برای همین می‌توان نامه‌های اداری، پرونده‌های شخصی و\nاسناد محرمانه را بی‌نگرانی به آن سپرد.\n\nمدل هوش مصنوعی بینا با هزاران صفحۀ چاپی و دست‌نویس آموزش دیده است. این\nمدل صفحه را به چند بخش تقسیم می‌کند، هر بخش را جداگانه می‌خواند و در پایان\nمتن کامل را به همان ترتیب خواندن کنار هم می‌گذارد.\n\nاگر رایانه کارت گرافیک انویدیا داشته باشد، بینا می‌تواند از آن کمک بگیرد و هر صفحه را\nچند برابر سریع‌تر بخواند. در غیر این صورت پردازندۀ مرکزی همین کار را با همان دقت\nانجام می‌دهد.", "lines": 11, "seconds": 18.71481087699067, "table_grids": 0, "table_errors": [], "precision_route": { "policy": "color-separated-handwriting-fp32-v1", "original_precision": false, "scope": { "protected": false, "reason": "No substantial coloured ink", "image_only": true } }, "character_parity": 1.0, "parity_gate": true }, { "image": "hand-plain.jpg", "text": "یک سال بعد از این کار دلیل این امر هم گستردگی\nمحصولات این شرکت بود این حافظه می‌تواند توسط نرم‌افزارهای\nارتباط مورد استفاده قرار بگیرند در این مجموعه دستوری از\nقواعد دستوری زیر استفاده می‌شود یک رشته می‌تواند شامل چندین\nدستور های س، باشد\nنرم‌افزار کاربردی نام عمومی برنامه‌های کامپیوتری برای\nانجام وظایف است نرم‌افزار منبع باز نرم‌افزاری، است\nکه کد منبع آن تحت یک مجوز خاص در اختیار\nدارندگان مجوز آن قرار می‌گیرد می‌وان از آن برای\nهر هدفی استفاده و، منتشر کرد کد منبع باز\nاست و، می‌توان آن را در صورت نیاز تغییر داد اگر\nهر دو نرم‌افزار از یک مجوز یکسان استفاده کنند معمولا\nقابل ادغام هستند مراقبت‌های بهداشتی و، کمک‌های\nاجتماعی برنامه‌های داخلی و، اختصاصی خط تجاری\nفعالیت این شرکت در راستای تولید نرم‌افزار و،\nبرنامه‌های کاربردی برای موبایل بود که در\nاین راه موفقیت بسیاری کسب کرده بود این سیستم عامل\nدر ابتدا ساختار بسیار ساده‌ای داشت و، از برنامه‌های\nمحدودی پشتیبانی می‌کرد اولین نسخه این سیستم عامل در", "lines": 19, "seconds": 18.373127989005297, "table_grids": 0, "table_errors": [], "precision_route": { "policy": "color-separated-handwriting-fp32-v1", "original_precision": false, "scope": { "protected": false, "reason": "No substantial coloured ink", "image_only": true } }, "character_parity": 0.9988950276243094, "parity_gate": true }, { "image": "hand-table.jpg", "text": "ردیف\nدبستان ......\nمدیریت آموزش وپرورش .....\nجمهوری اس لامی ایران\nاداره کل آموزش وپرورش استان .....\nفروردین\nوزارت آموزش وپرورش\nفصیل\nاولیا\nامضا آموزگار:\nجنگ جهانی یکم تا اواشی دهه ۱۹۳۰ خدمات هوایی\nنیروی رممن و، خدمات هوایی نیروی دریایی یونان\nبا قم یکی شده و، نیروی دریایی یویان را تشکیل\nدارند جنگ جهانی دوم و، جنگ داخلی نیروی\nهوایی یونان هران نظر وزارت دفاع ملی یونان و،\nمستقیما زن ظر ستاد کل دفاع ملی بونان قرار\nدا د\nقبل از جدو جهانی دوم استفاده بعد از\nجنگ جهانی بوم توکدات پر زاز هنگ\nجهانی ادوم استفاده گیسترده توسه سروهای انتظامی و،\nنظمی که از جنگ نیریک زمیمی دو، پلیس\nاستفاده توسط نیروی زمینی تا رشه ۱۹۶۰\nانستفاده پس از جنگ جهانی دوم درافت شده\nاز ایالات متحده\nسال تحصیلی ۰۳-۰۴\nنام ونام خانوادگی دانش آموز:\n۵\nماه\nمهر\nپایه پنجم\nقرآن\nبهمن\n۷\nاسفند\nورزش\nخوب\nخوب\nفصل\nمهر\nضعیف\nضکیف\nمتوسط\nکوبت دوم\n در سیس \nمهر وامضا مدیر مدرسه", "lines": 45, "seconds": 11.375215898966417, "table_grids": 0, "table_errors": [], "precision_route": { "policy": "color-separated-handwriting-fp32-v1", "original_precision": true, "scope": { "protected": true, "reason": "Confident separated colour layers", "image_only": true, "measurement_WH": [ 535, 768 ], "coloured_pixels": 39187, "separator_trace": { "reliable": true, "estimated_line_height": 19.2, "selected_class": 3, "classes": [ { "class_id": 0, "center": [ 0.33404725790023804, 0.3322349190711975, 0.3337506651878357 ], "pixels": 30314, "long_components": 152, "vertical_bins": 17, "score": 136.672 }, { "class_id": 1, "center": [ 0.43719831109046936, 0.4344765543937683, 0.12832078337669373 ], "pixels": 106, "long_components": 0, "vertical_bins": 0, "score": 0.0 }, { "class_id": 2, "center": [ 0.06904967129230499, 0.3027939200401306, 0.6281484961509705 ], "pixels": 1803, "long_components": 3, "vertical_bins": 1, "score": 13.005275715835074 }, { "class_id": 3, "center": [ 0.38381364941596985, 0.2830617427825928, 0.3331200182437897 ], "pixels": 17362, "long_components": 78, "vertical_bins": 12, "score": 649.6633543043666 } ], "pen_components": 358, "pen_pixels": 16746, "color_span_y": 481, "background_kernel": 23, "no_gold_or_template_used": true } } }, "character_parity": 1.0, "parity_gate": true } ], "cache_identity": "52e96c5c9cb3c75e81371614", "scope": "Public Bina Desktop samples; no claim of full gate certification", "readers": { "body": { "calls": 5, "hits": 0, "variant": "block16" }, "short_labels": { "calls": 0, "hits": 0, "variant": "block16" }, "printed": { "calls": 8, "hits": 0, "variant": "block16" }, "numeric_math": { "calls": 0, "hits": 0, "variant": "block16" }, "degraded": { "calls": 0, "hits": 0, "variant": "block16" } } } }, "full_gate_page_transcription_qualified": false, "android_hardware_tested": false, "ios_hardware_tested": false, "native_windows_tested": false, "pure_4bit_full_pipeline_qualified": false }