{ "generated_at": 1781326743.1110458, "note": "CER/WER vs .txt sidecar reference; field accuracy vs gt.json. sidecar = reference text source (CER\u22480 by construction).", "models": [ { "lab": "OpenBMB", "name": "MiniCPM-V-4.6", "params_b": 8.0, "size_gb_int4": 5.5, "modality": "vision-language (OCR + reasoning)", "role": "OCR backend + LLM extractor", "available": true }, { "lab": "OpenBMB", "name": "MiniCPM-o-4.5", "params_b": 8.0, "size_gb_int4": 5.5, "modality": "omni (vision/audio) VLM", "role": "alt OCR/VLM", "available": true }, { "lab": "OpenBMB", "name": "MiniCPM3-4B", "params_b": 4.0, "size_gb_int4": 2.8, "modality": "text LLM (reasoning + function-calling, 32k ctx)", "role": "ERP reasoning \u00b7 NLQ\u2192SQL \u00b7 report summarization (fine-tune target)", "available": true }, { "lab": "Cohere", "name": "Aya-Vision-8B", "params_b": 8.0, "size_gb_int4": 6.0, "modality": "vision-language (OCR/VQA, 23 langs)", "role": "OCR backend", "available": false }, { "lab": "Cohere", "name": "Aya-Vision-32B", "params_b": 32.0, "size_gb_int4": 18.0, "modality": "vision-language (OCR/VQA)", "role": "alt OCR backend (max-quality small)", "available": false }, { "lab": "Cohere", "name": "Command R7B", "params_b": 7.0, "size_gb_int4": 5.0, "modality": "text LLM (RAG + tool-use + reasoning, 128k ctx)", "role": "ERP RAG \u00b7 NLQ \u00b7 grounded reasoning", "available": false }, { "lab": "Black Forest Labs", "name": "FLUX.1 [dev]", "params_b": 12.0, "size_gb_int4": 12.0, "modality": "text-to-image GENERATION", "role": "synthetic test-document generator (not OCR)", "available": false }, { "lab": "Black Forest Labs", "name": "FLUX.1 [schnell]", "params_b": 12.0, "size_gb_int4": 12.0, "modality": "text-to-image GENERATION (fast)", "role": "synthetic test-document generator", "available": false } ], "backends": [ { "backend": "minicpm", "model": "MiniCPM-V-4.6", "params_b": 8.0, "size_gb": 5.5, "lab": "OpenBMB", "is_reference": false, "cer": 0.0262, "wer": 0.0876, "field_exact_match": 0.907, "field_f1": 0.9397, "avg_latency_ms": 6524.8167, "avg_cost_usd": 0.0002, "samples_scored": 6, "per_sample": [ { "sample": "invoice_scanned_basic", "cer": 0.0, "wer": 0.0, "field_exact": 0.889, "field_f1": 0.9, "latency_ms": 5560.8, "cost_usd": 0.0001952, "confidence": 0.7 }, { "sample": "receipt_scanned", "cer": 0.0942, "wer": 0.3103, "field_exact": 1.0, "field_f1": 1.0, "latency_ms": 4218.7, "cost_usd": 0.0001883, "confidence": 0.98 }, { "sample": "po_scanned", "cer": 0.0368, "wer": 0.1277, "field_exact": 1.0, "field_f1": 1.0, "latency_ms": 4404.9, "cost_usd": 0.0001835, "confidence": 0.98 }, { "sample": "contract_scanned", "cer": 0.0, "wer": 0.0, "field_exact": 0.636, "field_f1": 0.8, "latency_ms": 6532.2, "cost_usd": 0.000166, "confidence": 0.98 }, { "sample": "subscription_memo_scanned", "cer": 0.0, "wer": 0.0, "field_exact": 0.917, "field_f1": 0.938, "latency_ms": 5010.4, "cost_usd": 0.0001924, "confidence": 0.98 }, { "sample": "complex_invoice_messy", "cer": null, "wer": null, "field_exact": 1.0, "field_f1": 1.0, "latency_ms": 13421.9, "cost_usd": 0.0004414, "confidence": 0.98 } ] }, { "backend": "tesseract", "model": "tesseract", "params_b": null, "size_gb": null, "lab": "classic", "is_reference": false, "cer": 0.1468, "wer": 0.1848, "field_exact_match": 0.907, "field_f1": 0.9397, "avg_latency_ms": 3436.8667, "avg_cost_usd": 0.0001, "samples_scored": 6, "per_sample": [ { "sample": "invoice_scanned_basic", "cer": 0.1225, "wer": 0.1389, "field_exact": 0.889, "field_f1": 0.9, "latency_ms": 3698.6, "cost_usd": 0.0001242, "confidence": 0.68 }, { "sample": "receipt_scanned", "cer": 0.4555, "wer": 0.5172, "field_exact": 1.0, "field_f1": 1.0, "latency_ms": 2861.1, "cost_usd": 0.0001207, "confidence": 0.96 }, { "sample": "po_scanned", "cer": 0.0951, "wer": 0.1489, "field_exact": 1.0, "field_f1": 1.0, "latency_ms": 3390.6, "cost_usd": 0.000118, "confidence": 0.96 }, { "sample": "contract_scanned", "cer": 0.0, "wer": 0.0, "field_exact": 0.636, "field_f1": 0.8, "latency_ms": 2336.4, "cost_usd": 7.97e-05, "confidence": 0.96 }, { "sample": "subscription_memo_scanned", "cer": 0.061, "wer": 0.119, "field_exact": 0.917, "field_f1": 0.938, "latency_ms": 2243.8, "cost_usd": 0.0001211, "confidence": 0.96 }, { "sample": "complex_invoice_messy", "cer": null, "wer": null, "field_exact": 1.0, "field_f1": 1.0, "latency_ms": 6090.7, "cost_usd": 0.0002828, "confidence": 0.96 } ] }, { "backend": "sidecar", "model": "sidecar", "params_b": null, "size_gb": null, "lab": "classic", "is_reference": true, "cer": 0.0, "wer": 0.0, "field_exact_match": 0.907, "field_f1": 0.9397, "avg_latency_ms": 3235.3167, "avg_cost_usd": 0.0001, "samples_scored": 6, "per_sample": [ { "sample": "invoice_scanned_basic", "cer": 0.0, "wer": 0.0, "field_exact": 0.889, "field_f1": 0.9, "latency_ms": 1697.6, "cost_usd": 9.45e-05, "confidence": 0.66 }, { "sample": "receipt_scanned", "cer": 0.0, "wer": 0.0, "field_exact": 1.0, "field_f1": 1.0, "latency_ms": 2126.2, "cost_usd": 0.0001212, "confidence": 0.94 }, { "sample": "po_scanned", "cer": 0.0, "wer": 0.0, "field_exact": 1.0, "field_f1": 1.0, "latency_ms": 2194.4, "cost_usd": 0.0001184, "confidence": 0.94 }, { "sample": "contract_scanned", "cer": 0.0, "wer": 0.0, "field_exact": 0.636, "field_f1": 0.8, "latency_ms": 1522.1, "cost_usd": 7.97e-05, "confidence": 0.94 }, { "sample": "subscription_memo_scanned", "cer": 0.0, "wer": 0.0, "field_exact": 0.917, "field_f1": 0.938, "latency_ms": 1632.7, "cost_usd": 9.16e-05, "confidence": 0.94 }, { "sample": "complex_invoice_messy", "cer": null, "wer": null, "field_exact": 1.0, "field_f1": 1.0, "latency_ms": 10238.9, "cost_usd": 0.0002297, "confidence": 0.98 } ] } ], "best_ocr_quality": "minicpm", "best_document_analysis": "minicpm" }