backend/evals/ocr_quality_report.json

{
  "generated_at": 1781326743.1110458,
  "note": "CER/WER vs .txt sidecar reference; field accuracy vs gt.json. sidecar = reference text source (CER\u22480 by construction).",
  "models": [
    {
      "lab": "OpenBMB",
      "name": "MiniCPM-V-4.6",
      "params_b": 8.0,
      "size_gb_int4": 5.5,
      "modality": "vision-language (OCR + reasoning)",
      "role": "OCR backend + LLM extractor",
      "available": true
    },
    {
      "lab": "OpenBMB",
      "name": "MiniCPM-o-4.5",
      "params_b": 8.0,
      "size_gb_int4": 5.5,
      "modality": "omni (vision/audio) VLM",
      "role": "alt OCR/VLM",
      "available": true
    },
    {
      "lab": "OpenBMB",
      "name": "MiniCPM3-4B",
      "params_b": 4.0,
      "size_gb_int4": 2.8,
      "modality": "text LLM (reasoning + function-calling, 32k ctx)",
      "role": "ERP reasoning \u00b7 NLQ\u2192SQL \u00b7 report summarization (fine-tune target)",
      "available": true
    },
    {
      "lab": "Cohere",
      "name": "Aya-Vision-8B",
      "params_b": 8.0,
      "size_gb_int4": 6.0,
      "modality": "vision-language (OCR/VQA, 23 langs)",
      "role": "OCR backend",
      "available": false
    },
    {
      "lab": "Cohere",
      "name": "Aya-Vision-32B",
      "params_b": 32.0,
      "size_gb_int4": 18.0,
      "modality": "vision-language (OCR/VQA)",
      "role": "alt OCR backend (max-quality small)",
      "available": false
    },
    {
      "lab": "Cohere",
      "name": "Command R7B",
      "params_b": 7.0,
      "size_gb_int4": 5.0,
      "modality": "text LLM (RAG + tool-use + reasoning, 128k ctx)",
      "role": "ERP RAG \u00b7 NLQ \u00b7 grounded reasoning",
      "available": false
    },
    {
      "lab": "Black Forest Labs",
      "name": "FLUX.1 [dev]",
      "params_b": 12.0,
      "size_gb_int4": 12.0,
      "modality": "text-to-image GENERATION",
      "role": "synthetic test-document generator (not OCR)",
      "available": false
    },
    {
      "lab": "Black Forest Labs",
      "name": "FLUX.1 [schnell]",
      "params_b": 12.0,
      "size_gb_int4": 12.0,
      "modality": "text-to-image GENERATION (fast)",
      "role": "synthetic test-document generator",
      "available": false
    }
  ],
  "backends": [
    {
      "backend": "minicpm",
      "model": "MiniCPM-V-4.6",
      "params_b": 8.0,
      "size_gb": 5.5,
      "lab": "OpenBMB",
      "is_reference": false,
      "cer": 0.0262,
      "wer": 0.0876,
      "field_exact_match": 0.907,
      "field_f1": 0.9397,
      "avg_latency_ms": 6524.8167,
      "avg_cost_usd": 0.0002,
      "samples_scored": 6,
      "per_sample": [
        {
          "sample": "invoice_scanned_basic",
          "cer": 0.0,
          "wer": 0.0,
          "field_exact": 0.889,
          "field_f1": 0.9,
          "latency_ms": 5560.8,
          "cost_usd": 0.0001952,
          "confidence": 0.7
        },
        {
          "sample": "receipt_scanned",
          "cer": 0.0942,
          "wer": 0.3103,
          "field_exact": 1.0,
          "field_f1": 1.0,
          "latency_ms": 4218.7,
          "cost_usd": 0.0001883,
          "confidence": 0.98
        },
        {
          "sample": "po_scanned",
          "cer": 0.0368,
          "wer": 0.1277,
          "field_exact": 1.0,
          "field_f1": 1.0,
          "latency_ms": 4404.9,
          "cost_usd": 0.0001835,
          "confidence": 0.98
        },
        {
          "sample": "contract_scanned",
          "cer": 0.0,
          "wer": 0.0,
          "field_exact": 0.636,
          "field_f1": 0.8,
          "latency_ms": 6532.2,
          "cost_usd": 0.000166,
          "confidence": 0.98
        },
        {
          "sample": "subscription_memo_scanned",
          "cer": 0.0,
          "wer": 0.0,
          "field_exact": 0.917,
          "field_f1": 0.938,
          "latency_ms": 5010.4,
          "cost_usd": 0.0001924,
          "confidence": 0.98
        },
        {
          "sample": "complex_invoice_messy",
          "cer": null,
          "wer": null,
          "field_exact": 1.0,
          "field_f1": 1.0,
          "latency_ms": 13421.9,
          "cost_usd": 0.0004414,
          "confidence": 0.98
        }
      ]
    },
    {
      "backend": "tesseract",
      "model": "tesseract",
      "params_b": null,
      "size_gb": null,
      "lab": "classic",
      "is_reference": false,
      "cer": 0.1468,
      "wer": 0.1848,
      "field_exact_match": 0.907,
      "field_f1": 0.9397,
      "avg_latency_ms": 3436.8667,
      "avg_cost_usd": 0.0001,
      "samples_scored": 6,
      "per_sample": [
        {
          "sample": "invoice_scanned_basic",
          "cer": 0.1225,
          "wer": 0.1389,
          "field_exact": 0.889,
          "field_f1": 0.9,
          "latency_ms": 3698.6,
          "cost_usd": 0.0001242,
          "confidence": 0.68
        },
        {
          "sample": "receipt_scanned",
          "cer": 0.4555,
          "wer": 0.5172,
          "field_exact": 1.0,
          "field_f1": 1.0,
          "latency_ms": 2861.1,
          "cost_usd": 0.0001207,
          "confidence": 0.96
        },
        {
          "sample": "po_scanned",
          "cer": 0.0951,
          "wer": 0.1489,
          "field_exact": 1.0,
          "field_f1": 1.0,
          "latency_ms": 3390.6,
          "cost_usd": 0.000118,
          "confidence": 0.96
        },
        {
          "sample": "contract_scanned",
          "cer": 0.0,
          "wer": 0.0,
          "field_exact": 0.636,
          "field_f1": 0.8,
          "latency_ms": 2336.4,
          "cost_usd": 7.97e-05,
          "confidence": 0.96
        },
        {
          "sample": "subscription_memo_scanned",
          "cer": 0.061,
          "wer": 0.119,
          "field_exact": 0.917,
          "field_f1": 0.938,
          "latency_ms": 2243.8,
          "cost_usd": 0.0001211,
          "confidence": 0.96
        },
        {
          "sample": "complex_invoice_messy",
          "cer": null,
          "wer": null,
          "field_exact": 1.0,
          "field_f1": 1.0,
          "latency_ms": 6090.7,
          "cost_usd": 0.0002828,
          "confidence": 0.96
        }
      ]
    },
    {
      "backend": "sidecar",
      "model": "sidecar",
      "params_b": null,
      "size_gb": null,
      "lab": "classic",
      "is_reference": true,
      "cer": 0.0,
      "wer": 0.0,
      "field_exact_match": 0.907,
      "field_f1": 0.9397,
      "avg_latency_ms": 3235.3167,
      "avg_cost_usd": 0.0001,
      "samples_scored": 6,
      "per_sample": [
        {
          "sample": "invoice_scanned_basic",
          "cer": 0.0,
          "wer": 0.0,
          "field_exact": 0.889,
          "field_f1": 0.9,
          "latency_ms": 1697.6,
          "cost_usd": 9.45e-05,
          "confidence": 0.66
        },
        {
          "sample": "receipt_scanned",
          "cer": 0.0,
          "wer": 0.0,
          "field_exact": 1.0,
          "field_f1": 1.0,
          "latency_ms": 2126.2,
          "cost_usd": 0.0001212,
          "confidence": 0.94
        },
        {
          "sample": "po_scanned",
          "cer": 0.0,
          "wer": 0.0,
          "field_exact": 1.0,
          "field_f1": 1.0,
          "latency_ms": 2194.4,
          "cost_usd": 0.0001184,
          "confidence": 0.94
        },
        {
          "sample": "contract_scanned",
          "cer": 0.0,
          "wer": 0.0,
          "field_exact": 0.636,
          "field_f1": 0.8,
          "latency_ms": 1522.1,
          "cost_usd": 7.97e-05,
          "confidence": 0.94
        },
        {
          "sample": "subscription_memo_scanned",
          "cer": 0.0,
          "wer": 0.0,
          "field_exact": 0.917,
          "field_f1": 0.938,
          "latency_ms": 1632.7,
          "cost_usd": 9.16e-05,
          "confidence": 0.94
        },
        {
          "sample": "complex_invoice_messy",
          "cer": null,
          "wer": null,
          "field_exact": 1.0,
          "field_f1": 1.0,
          "latency_ms": 10238.9,
          "cost_usd": 0.0002297,
          "confidence": 0.98
        }
      ]
    }
  ],
  "best_ocr_quality": "minicpm",
  "best_document_analysis": "minicpm"
}